From bf038fb069975c812fd92ac80a598fdb838e2c98 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Fri, 17 Nov 2017 17:28:15 -0800 Subject: [PATCH] HIVE-18100 : Some tests time out --- .../test/resources/testconfiguration.properties | 7 +- .../test/queries/clientpositive/explainanalyze_1.q | 8 + .../test/queries/clientpositive/explainanalyze_2.q | 8 - .../test/queries/clientpositive/unionDistinct_1.q | 315 +- .../test/queries/clientpositive/unionDistinct_2.q | 23 - .../test/queries/clientpositive/unionDistinct_3.q | 182 + .../{tez => llap}/explainanalyze_2.q.out | 326 +- .../clientpositive/llap/unionDistinct_1.q.out | 22877 ++++++++----------- .../clientpositive/llap/unionDistinct_2.q.out | 108 - .../clientpositive/llap/unionDistinct_3.q.out | 2650 +++ .../clientpositive/tez/explainanalyze_1.q.out | 58 + .../clientpositive/tez/unionDistinct_2.q.out | 553 - 12 files changed, 12207 insertions(+), 14908 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/unionDistinct_3.q rename ql/src/test/results/clientpositive/{tez => llap}/explainanalyze_2.q.out (93%) create mode 100644 ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out delete mode 100644 ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 5d17733cf4..ab3cd43158 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -43,7 +43,6 @@ disabled.query.files=ql_rewrite_gbtoidx.q,\ minitez.query.files.shared=delete_orig_table.q,\ orc_merge12.q,\ orc_vectorization_ppd.q,\ - unionDistinct_2.q,\ update_orig_table.q,\ vector_join_part_col_char.q,\ vector_non_string_partition.q,\ @@ -55,7 +54,6 @@ minitez.query.files.shared=delete_orig_table.q,\ minitez.query.files=acid_vectorization_original_tez.q,\ explainuser_3.q,\ explainanalyze_1.q,\ - explainanalyze_2.q,\ explainanalyze_3.q,\ explainanalyze_4.q,\ explainanalyze_5.q,\ @@ -80,7 +78,6 @@ minillap.shared.query.files=insert_into1.q,\ orc_merge_diff_fs.q,\ parallel_colstats.q,\ parquet_types_vectorization.q,\ - unionDistinct_1.q,\ union_type_chk.q,\ cte_2.q,\ cte_4.q,\ @@ -467,7 +464,6 @@ minillap.query.files=acid_bucket_pruning.q,\ temp_table_external.q,\ table_nonprintable.q,\ llap_nullscan.q,\ - unionDistinct_1.q,\ rcfile_merge2.q,\ rcfile_merge3.q,\ rcfile_merge4.q,\ @@ -477,6 +473,7 @@ minillap.query.files=acid_bucket_pruning.q,\ dynamic_partition_pruning_2.q,\ tez_union_dynamic_partition.q,\ tez_union_dynamic_partition_2.q,\ + unionDistinct_1.q,\ load_fs2.q,\ llap_stats.q,\ multi_count_distinct_null.q @@ -540,6 +537,7 @@ minillaplocal.query.files=\ escape1.q,\ escape2.q,\ exchgpartition2lel.q,\ + explainanalyze_2.q,\ explainuser_1.q,\ explainuser_4.q,\ groupby2.q,\ @@ -801,6 +799,7 @@ minillaplocal.query.files=\ smb_mapjoin_17.q,\ groupby_resolution.q,\ windowing_windowspec2.q,\ + unionDistinct_3.q,\ vectorized_join46.q,\ vectorized_multi_output_select.q,\ partialdhj.q diff --git a/ql/src/test/queries/clientpositive/explainanalyze_1.q b/ql/src/test/queries/clientpositive/explainanalyze_1.q index 0a33a4f89c..6611922016 100644 --- a/ql/src/test/queries/clientpositive/explainanalyze_1.q +++ b/ql/src/test/queries/clientpositive/explainanalyze_1.q @@ -37,3 +37,11 @@ set hive.auto.convert.join.noconditionaltask.size=10000; EXPLAIN analyze SELECT x.key, y.value FROM src x JOIN src y ON (x.key = y.key); + +set hive.entity.capture.transform=true; +explain analyze +SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key; + diff --git a/ql/src/test/queries/clientpositive/explainanalyze_2.q b/ql/src/test/queries/clientpositive/explainanalyze_2.q index b764d7d2af..dd515ec5cc 100644 --- a/ql/src/test/queries/clientpositive/explainanalyze_2.q +++ b/ql/src/test/queries/clientpositive/explainanalyze_2.q @@ -1,7 +1,6 @@ set hive.map.aggr=false; set hive.strict.checks.bucketing=false; - set hive.explain.user=true; explain analyze @@ -139,13 +138,6 @@ INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT explain analyze FROM UNIQUEJOIN PRESERVE src a (a.key), PRESERVE src1 b (b.key), PRESERVE srcpart c (c.key) SELECT a.key, b.key, c.key; -set hive.entity.capture.transform=true; - -explain analyze -SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key; explain analyze FROM ( diff --git a/ql/src/test/queries/clientpositive/unionDistinct_1.q b/ql/src/test/queries/clientpositive/unionDistinct_1.q index f2a0dc17d1..4978b563af 100644 --- a/ql/src/test/queries/clientpositive/unionDistinct_1.q +++ b/ql/src/test/queries/clientpositive/unionDistinct_1.q @@ -1,3 +1,4 @@ +set hive.llap.execution.mode=auto; set hive.mapred.mode=nonstrict; set hive.explain.user=false; -- SORT_QUERY_RESULTS @@ -27,27 +28,6 @@ insert overwrite table tmptable select * from tmptable x sort by x.key; --- union11.q - - --- SORT_BEFORE_DIFF --- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; - - - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; - - -- union12.q @@ -85,110 +65,6 @@ explain select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc; --- union14.q - - --- SORT_BEFORE_DIFF --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key; - - - - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key; --- union15.q - - --- SORT_BEFORE_DIFF --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; - - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; - - --- union16.q - --- SORT_BEFORE_DIFF -EXPLAIN -SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src; - - -SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src; -- union17.q CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE; @@ -261,76 +137,6 @@ SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value; SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2; - --- union2.q - --- SORT_BEFORE_DIFF --- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink - -explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc; - -select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc; --- union20.q - --- SORT_BEFORE_DIFF --- union :map-reduce sub-queries followed by join - -explain -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key); - -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key); --- union21.q - --- SORT_BEFORE_DIFF --- union of constants, udf outputs, and columns from text table and thrift table - -explain -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key; - -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key; -- union22.q -- SORT_QUERY_RESULTS @@ -819,11 +625,8 @@ insert overwrite table t6 select * from t5; select * from t6; -drop table t1; -drop table t2; -create table t1 as select * from src where key < 10; -create table t2 as select key, count(1) as cnt from src where key < 10 group by key; +create table t9 as select key, count(1) as cnt from src where key < 10 group by key; create table t7(c1 string, cnt int); create table t8(c1 string, cnt int); @@ -833,7 +636,7 @@ from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT - select key as c1, cnt from t2 + select key as c1, cnt from t9 ) x insert overwrite table t7 select c1, count(1) group by c1 @@ -844,7 +647,7 @@ from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT - select key as c1, cnt from t2 + select key as c1, cnt from t9 ) x insert overwrite table t7 select c1, count(1) group by c1 @@ -860,13 +663,6 @@ select * from t8; -- This tests various union queries which have columns on one side of the query -- being of double type and those on the other side another -drop table if exists t1; - -drop table if exists t2; - -CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10; -CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10; - -- Test simple union with double EXPLAIN SELECT * FROM @@ -1035,108 +831,5 @@ SELECT * FROM ( UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1; --- union4.q - - - --- union case: both subqueries are map-reduce jobs on same input, followed by filesink - -drop table if exists tmptable; - -create table tmptable(key string, value int); - -explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc; - -insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc; - -select * from tmptable x sort by x.key; - - --- union5.q - - --- SORT_BEFORE_DIFF --- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key; - -select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key; --- union6.q - - --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink - -drop table if exists tmptable; - -create table tmptable(key string, value string); - -explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc; - -insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc; - -select * from tmptable x sort by x.key, x.value; - - --- union7.q - - - --- SORT_BEFORE_DIFF --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key; - -select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key; - - --- union8.q - --- SORT_BEFORE_DIFF --- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink - -explain - select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; - -select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; --- union9.q - --- SORT_BEFORE_DIFF --- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink - -explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; diff --git a/ql/src/test/queries/clientpositive/unionDistinct_2.q b/ql/src/test/queries/clientpositive/unionDistinct_2.q index 324bf68f58..da47d38b7b 100644 --- a/ql/src/test/queries/clientpositive/unionDistinct_2.q +++ b/ql/src/test/queries/clientpositive/unionDistinct_2.q @@ -6,21 +6,6 @@ CREATE TABLE u2 as select key, value from src order by key limit 3; CREATE TABLE u3 as select key, value from src order by key desc limit 5; -select * from u1; - -select * from u2; - -select * from u3; - -select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab; - select key, value from ( select key, value from u1 @@ -57,14 +42,6 @@ union select key as key, value from u3 ) tab; -select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab; select distinct * from ( diff --git a/ql/src/test/queries/clientpositive/unionDistinct_3.q b/ql/src/test/queries/clientpositive/unionDistinct_3.q new file mode 100644 index 0000000000..e3cf236928 --- /dev/null +++ b/ql/src/test/queries/clientpositive/unionDistinct_3.q @@ -0,0 +1,182 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; + +-- union2.q + +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; + +select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; + +-- union6.q + + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table if exists tmptable; + +create table tmptable(key string, value string); + +explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +select * from tmptable x sort by x.key, x.value; + +drop table if exists tmptable; + + +-- union8.q + +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; + +select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; + +-- union11.q + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + + + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + +-- union14.q + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key; + + + + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key; +-- union15.q + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; + + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; + +-- union16.q + +EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src; + + +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src; + +-- union20.q + +-- union :map-reduce sub-queries followed by join + +explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key); + +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key); +-- union21.q + +-- union of constants, udf outputs, and columns from text table and thrift table + +explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key; + +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key; + diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out similarity index 93% rename from ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out rename to ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out index e5b1d74654..7db3b13bbf 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out @@ -57,19 +57,19 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 + Reducer 7 llap File Output Operator [FS_56] Group By Operator [GBY_54] (rows=32/15 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 6 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] + <-Reducer 11 [CONTAINS] llap Reduce Output Operator [RS_53] PartitionCols:_col0, _col1 Select Operator [SEL_49] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_82] (rows=67/61 width=177) Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] llap SHUFFLE [RS_47] PartitionCols:_col0 Select Operator [SEL_42] (rows=500/500 width=87) @@ -78,12 +78,12 @@ Stage-0 predicate:key is not null TableScan [TS_40] (rows=500/500 width=87) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 10 [SIMPLE_EDGE] + <-Reducer 10 [SIMPLE_EDGE] llap SHUFFLE [RS_46] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_81] (rows=42/52 width=177) Conds:RS_43._col1=RS_44._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] + <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_44] PartitionCols:_col1 Select Operator [SEL_14] (rows=25/25 width=175) @@ -92,7 +92,7 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 15 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] llap SHUFFLE [RS_43] PartitionCols:_col1 Select Operator [SEL_36] (rows=525/319 width=178) @@ -100,7 +100,7 @@ Stage-0 Group By Operator [GBY_35] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] + <-Map 13 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_27] (rows=25/25 width=175) @@ -109,7 +109,7 @@ Stage-0 predicate:value is not null TableScan [TS_25] (rows=25/25 width=175) Output:["key","value"] - <-Map 16 [CONTAINS] + <-Map 16 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_30] (rows=500/500 width=178) @@ -118,14 +118,14 @@ Stage-0 predicate:value is not null TableScan [TS_28] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] + <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_53] PartitionCols:_col0, _col1 Select Operator [SEL_24] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_80] (rows=67/61 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col2"] - <-Map 12 [SIMPLE_EDGE] + <-Map 12 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col0 Select Operator [SEL_17] (rows=500/500 width=87) @@ -134,16 +134,16 @@ Stage-0 predicate:key is not null TableScan [TS_15] (rows=500/500 width=87) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_79] (rows=42/52 width=177) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] + <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 Select Operator [SEL_11] (rows=525/319 width=178) @@ -151,7 +151,7 @@ Stage-0 Group By Operator [GBY_10] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_2] (rows=25/25 width=175) @@ -160,7 +160,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25/25 width=175) Output:["key","value"] - <-Map 8 [CONTAINS] + <-Map 8 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_5] (rows=500/500 width=178) @@ -257,19 +257,19 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 + Reducer 9 llap File Output Operator [FS_114] Group By Operator [GBY_112] (rows=398/15 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 8 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] + <-Reducer 15 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Select Operator [SEL_107] (rows=199/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_162] (rows=199/61 width=177) Conds:RS_104._col2=RS_105._col0(Inner),Output:["_col2","_col5"] - <-Map 16 [SIMPLE_EDGE] + <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_105] PartitionCols:_col0 Select Operator [SEL_17] (rows=500/500 width=178) @@ -278,12 +278,12 @@ Stage-0 predicate:key is not null TableScan [TS_15] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 14 [SIMPLE_EDGE] + <-Reducer 14 [SIMPLE_EDGE] llap SHUFFLE [RS_104] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_161] (rows=123/52 width=86) Conds:RS_101._col1=RS_102._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_102] PartitionCols:_col1 Select Operator [SEL_14] (rows=25/25 width=175) @@ -292,7 +292,7 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] + <-Reducer 31 [SIMPLE_EDGE] llap SHUFFLE [RS_101] PartitionCols:_col1 Select Operator [SEL_94] (rows=1525/319 width=178) @@ -300,7 +300,7 @@ Stage-0 Group By Operator [GBY_93] (rows=1525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] + <-Map 34 [CONTAINS] llap Reduce Output Operator [RS_92] PartitionCols:_col1, _col0 Select Operator [SEL_88] (rows=500/500 width=178) @@ -309,7 +309,7 @@ Stage-0 predicate:value is not null TableScan [TS_86] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 29 [CONTAINS] + <-Reducer 29 [CONTAINS] llap Reduce Output Operator [RS_92] PartitionCols:_col1, _col0 Select Operator [SEL_85] (rows=1025/319 width=178) @@ -317,7 +317,7 @@ Stage-0 Group By Operator [GBY_84] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] + <-Map 33 [CONTAINS] llap Reduce Output Operator [RS_83] PartitionCols:_col1, _col0 Select Operator [SEL_79] (rows=500/500 width=178) @@ -326,7 +326,7 @@ Stage-0 predicate:value is not null TableScan [TS_77] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 27 [CONTAINS] + <-Reducer 27 [CONTAINS] llap Reduce Output Operator [RS_83] PartitionCols:_col1, _col0 Select Operator [SEL_76] (rows=525/319 width=178) @@ -334,7 +334,7 @@ Stage-0 Group By Operator [GBY_75] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] + <-Map 25 [CONTAINS] llap Reduce Output Operator [RS_74] PartitionCols:_col1, _col0 Select Operator [SEL_67] (rows=25/25 width=175) @@ -343,7 +343,7 @@ Stage-0 predicate:value is not null TableScan [TS_65] (rows=25/25 width=175) Output:["key","value"] - <-Map 32 [CONTAINS] + <-Map 32 [CONTAINS] llap Reduce Output Operator [RS_74] PartitionCols:_col1, _col0 Select Operator [SEL_70] (rows=500/500 width=178) @@ -352,20 +352,20 @@ Stage-0 predicate:value is not null TableScan [TS_68] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 7 [CONTAINS] + <-Reducer 7 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Group By Operator [GBY_63] (rows=199/15 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 6 [SIMPLE_EDGE] - <-Reducer 13 [CONTAINS] + <-Reducer 13 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_58] (rows=132/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_160] (rows=132/61 width=177) Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] - <-Map 24 [SIMPLE_EDGE] + <-Map 24 [SIMPLE_EDGE] llap SHUFFLE [RS_56] PartitionCols:_col0 Select Operator [SEL_51] (rows=500/500 width=178) @@ -374,16 +374,16 @@ Stage-0 predicate:key is not null TableScan [TS_49] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 12 [SIMPLE_EDGE] llap SHUFFLE [RS_55] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_159] (rows=82/52 width=86) Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_53] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 21 [SIMPLE_EDGE] + <-Reducer 21 [SIMPLE_EDGE] llap SHUFFLE [RS_52] PartitionCols:_col1 Select Operator [SEL_45] (rows=1025/319 width=178) @@ -391,7 +391,7 @@ Stage-0 Group By Operator [GBY_44] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 20 [SIMPLE_EDGE] - <-Map 23 [CONTAINS] + <-Map 23 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_39] (rows=500/500 width=178) @@ -400,7 +400,7 @@ Stage-0 predicate:value is not null TableScan [TS_37] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 19 [CONTAINS] + <-Reducer 19 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_36] (rows=525/319 width=178) @@ -408,7 +408,7 @@ Stage-0 Group By Operator [GBY_35] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 18 [SIMPLE_EDGE] - <-Map 17 [CONTAINS] + <-Map 17 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_27] (rows=25/25 width=175) @@ -417,7 +417,7 @@ Stage-0 predicate:value is not null TableScan [TS_25] (rows=25/25 width=175) Output:["key","value"] - <-Map 22 [CONTAINS] + <-Map 22 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_30] (rows=500/500 width=178) @@ -426,27 +426,27 @@ Stage-0 predicate:value is not null TableScan [TS_28] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] + <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_24] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_158] (rows=67/61 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] - <-Map 16 [SIMPLE_EDGE] + <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_17] - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_157] (rows=42/52 width=86) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 Select Operator [SEL_11] (rows=525/319 width=178) @@ -454,7 +454,7 @@ Stage-0 Group By Operator [GBY_10] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_2] (rows=25/25 width=175) @@ -463,7 +463,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25/25 width=175) Output:["key","value"] - <-Map 10 [CONTAINS] + <-Map 10 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_5] (rows=500/500 width=178) @@ -627,7 +627,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 + Map 1 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=253/480 width=190) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -669,13 +669,13 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 llap File Output Operator [FS_16] Select Operator [SEL_15] (rows=278/1166 width=3) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_27] (rows=278/1166 width=3) Conds:RS_12._col2=RS_13._col1(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_25] (rows=253/480 width=3) @@ -692,7 +692,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] + <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=230/242 width=175) @@ -728,7 +728,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 + Map 1 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=253/480 width=190) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -774,13 +774,13 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 llap File Output Operator [FS_16] Select Operator [SEL_15] (rows=278/1166 width=3) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_27] (rows=278/1166 width=3) Conds:RS_12._col2=RS_13._col1(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_25] (rows=253/480 width=3) @@ -797,7 +797,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] + <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=230/242 width=175) @@ -851,15 +851,15 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 + Reducer 4 llap File Output Operator [FS_25] Group By Operator [GBY_23] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] + <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_22] Merge Join Operator [MERGEJOIN_38] (rows=531/1646 width=3) Conds:Union 2._col0=RS_19._col0(Inner) - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_17] (rows=475/500 width=3) @@ -869,7 +869,7 @@ Stage-0 TableScan [TS_15] (rows=500/500 width=3) default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_18] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_36] (rows=253/480 width=3) @@ -886,7 +886,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) Output:["key"] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_18] PartitionCols:_col0 Select Operator [SEL_12] (rows=230/242 width=3) @@ -940,15 +940,15 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 + Reducer 5 llap File Output Operator [FS_31] Group By Operator [GBY_29] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] + <-Reducer 4 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_28] Merge Join Operator [MERGEJOIN_50] (rows=558/3768 width=3) Conds:Union 3._col0=RS_25._col0(Inner) - <-Map 9 [SIMPLE_EDGE] + <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_25] PartitionCols:_col0 Select Operator [SEL_23] (rows=475/500 width=3) @@ -958,7 +958,7 @@ Stage-0 TableScan [TS_21] (rows=500/500 width=3) default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] + <-Map 8 [CONTAINS] llap Reduce Output Operator [RS_24] PartitionCols:_col0 Select Operator [SEL_18] (rows=230/242 width=3) @@ -967,14 +967,14 @@ Stage-0 predicate:key is not null TableScan [TS_16] (rows=242/242 width=3) Output:["key"] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_24] PartitionCols:_col0 Select Operator [SEL_15] (rows=278/1166 width=3) Output:["_col0"] Merge Join Operator [MERGEJOIN_49] (rows=278/1166 width=3) Conds:RS_12._col2=RS_13._col1(Inner),Output:["_col1"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_47] (rows=253/480 width=3) @@ -991,7 +991,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=230/242 width=175) @@ -1137,14 +1137,14 @@ Stage-5 Dependency Collection{} Stage-3 Union 5 - <-Reducer 12 [CONTAINS] + <-Reducer 12 [CONTAINS] llap File Output Operator [FS_75] table:{"name:":"default.a"} Select Operator [SEL_44] (rows=2682/5421 width=178) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_122] (rows=2682/5421 width=178) Conds:RS_41._col1=RS_42._col0(Inner),Output:["_col1","_col4"] - <-Map 17 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] llap SHUFFLE [RS_42] PartitionCols:_col0 Select Operator [SEL_37] (rows=500/500 width=178) @@ -1153,12 +1153,12 @@ Stage-5 predicate:key is not null TableScan [TS_35] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 11 [SIMPLE_EDGE] + <-Reducer 11 [SIMPLE_EDGE] llap SHUFFLE [RS_41] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_121] (rows=1658/2097 width=87) Conds:Union 14._col0=RS_39._col1(Inner),Output:["_col1"] - <-Map 10 [SIMPLE_EDGE] + <-Map 10 [SIMPLE_EDGE] llap SHUFFLE [RS_39] PartitionCols:_col1 Select Operator [SEL_34] (rows=500/500 width=178) @@ -1168,7 +1168,7 @@ Stage-5 TableScan [TS_11] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] + <-Map 13 [CONTAINS] llap Reduce Output Operator [RS_38] PartitionCols:_col0 Select Operator [SEL_23] (rows=25/25 width=89) @@ -1177,7 +1177,7 @@ Stage-5 predicate:value is not null TableScan [TS_21] (rows=25/25 width=89) Output:["value"] - <-Map 15 [CONTAINS] + <-Map 15 [CONTAINS] llap Reduce Output Operator [RS_38] PartitionCols:_col0 Select Operator [SEL_26] (rows=500/500 width=91) @@ -1186,7 +1186,7 @@ Stage-5 predicate:value is not null TableScan [TS_24] (rows=500/500 width=91) Output:["value"] - <-Map 16 [CONTAINS] + <-Map 16 [CONTAINS] llap Reduce Output Operator [RS_38] PartitionCols:_col0 Select Operator [SEL_30] (rows=500/500 width=91) @@ -1201,14 +1201,14 @@ Stage-5 File Output Operator [FS_79] table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_44] - <-Reducer 4 [CONTAINS] + <-Reducer 4 [CONTAINS] llap File Output Operator [FS_75] table:{"name:":"default.a"} Select Operator [SEL_20] (rows=67/170 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_120] (rows=67/170 width=177) Conds:RS_17._col1=RS_18._col0(Inner),Output:["_col1","_col4"] - <-Map 10 [SIMPLE_EDGE] + <-Map 10 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col0 Select Operator [SEL_13] (rows=500/500 width=178) @@ -1216,12 +1216,12 @@ Stage-5 Filter Operator [FIL_107] (rows=500/500 width=178) predicate:key is not null Please refer to the previous TableScan [TS_11] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_17] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_119] (rows=42/108 width=86) Conds:Union 2._col0=RS_15._col1(Inner),Output:["_col1"] - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_15] PartitionCols:_col1 Select Operator [SEL_10] (rows=25/25 width=175) @@ -1231,7 +1231,7 @@ Stage-5 TableScan [TS_8] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_14] PartitionCols:_col0 Select Operator [SEL_2] (rows=25/25 width=89) @@ -1240,7 +1240,7 @@ Stage-5 predicate:value is not null TableScan [TS_0] (rows=25/25 width=89) Output:["value"] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_14] PartitionCols:_col0 Select Operator [SEL_5] (rows=500/500 width=91) @@ -1255,23 +1255,23 @@ Stage-5 File Output Operator [FS_79] table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_20] - <-Reducer 9 [CONTAINS] + <-Reducer 9 [CONTAINS] llap File Output Operator [FS_75] table:{"name:":"default.a"} Select Operator [SEL_72] (rows=192/820 width=175) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_124] (rows=192/820 width=175) Conds:RS_69._col1=Union 20._col0(Inner),Output:["_col0","_col3"] - <-Reducer 8 [SIMPLE_EDGE] + <-Reducer 8 [SIMPLE_EDGE] llap SHUFFLE [RS_69] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_123] (rows=39/115 width=264) Conds:RS_66._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_66] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_10] - <-Map 18 [SIMPLE_EDGE] + <-Map 18 [SIMPLE_EDGE] llap SHUFFLE [RS_67] PartitionCols:_col0 Select Operator [SEL_51] (rows=25/25 width=175) @@ -1281,7 +1281,7 @@ Stage-5 TableScan [TS_49] (rows=25/25 width=175) default@src1,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Union 20 [SIMPLE_EDGE] - <-Map 19 [CONTAINS] + <-Map 19 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_54] (rows=25/25 width=89) @@ -1290,7 +1290,7 @@ Stage-5 predicate:value is not null TableScan [TS_52] (rows=25/25 width=89) Output:["value"] - <-Map 21 [CONTAINS] + <-Map 21 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_57] (rows=500/500 width=91) @@ -1299,7 +1299,7 @@ Stage-5 predicate:value is not null TableScan [TS_55] (rows=500/500 width=91) Output:["value"] - <-Map 22 [CONTAINS] + <-Map 22 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_61] (rows=500/500 width=91) @@ -1308,7 +1308,7 @@ Stage-5 predicate:value is not null TableScan [TS_59] (rows=500/500 width=91) Output:["value"] - <-Map 23 [CONTAINS] + <-Map 23 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_64] (rows=500/500 width=91) @@ -1443,25 +1443,25 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 - Reducer 9 + Reducer 9 llap File Output Operator [FS_115] table:{"name:":"default.a"} Group By Operator [GBY_112] (rows=2941/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 8 [SIMPLE_EDGE] - <-Reducer 13 [CONTAINS] + <-Reducer 13 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Select Operator [SEL_107] (rows=192/304 width=175) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_164] (rows=192/304 width=175) Conds:RS_104._col1=RS_105._col1(Inner),Output:["_col0","_col3"] - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 12 [SIMPLE_EDGE] llap SHUFFLE [RS_104] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_163] (rows=39/115 width=264) Conds:RS_101._col0=RS_102._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_101] PartitionCols:_col0 Select Operator [SEL_14] (rows=25/25 width=175) @@ -1470,7 +1470,7 @@ Stage-5 predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 25 [SIMPLE_EDGE] + <-Map 25 [SIMPLE_EDGE] llap SHUFFLE [RS_102] PartitionCols:_col0 Select Operator [SEL_70] (rows=25/25 width=175) @@ -1479,7 +1479,7 @@ Stage-5 predicate:key is not null TableScan [TS_68] (rows=25/25 width=175) default@src1,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 32 [SIMPLE_EDGE] + <-Reducer 32 [SIMPLE_EDGE] llap SHUFFLE [RS_105] PartitionCols:_col1 Select Operator [SEL_100] (rows=1525/319 width=178) @@ -1487,7 +1487,7 @@ Stage-5 Group By Operator [GBY_99] (rows=1525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 31 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] + <-Map 35 [CONTAINS] llap Reduce Output Operator [RS_98] PartitionCols:_col1, _col0 Select Operator [SEL_94] (rows=500/500 width=178) @@ -1496,7 +1496,7 @@ Stage-5 predicate:value is not null TableScan [TS_92] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 30 [CONTAINS] + <-Reducer 30 [CONTAINS] llap Reduce Output Operator [RS_98] PartitionCols:_col1, _col0 Select Operator [SEL_91] (rows=1025/319 width=178) @@ -1504,7 +1504,7 @@ Stage-5 Group By Operator [GBY_90] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 29 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] + <-Map 34 [CONTAINS] llap Reduce Output Operator [RS_89] PartitionCols:_col1, _col0 Select Operator [SEL_85] (rows=500/500 width=178) @@ -1513,7 +1513,7 @@ Stage-5 predicate:value is not null TableScan [TS_83] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 28 [CONTAINS] + <-Reducer 28 [CONTAINS] llap Reduce Output Operator [RS_89] PartitionCols:_col1, _col0 Select Operator [SEL_82] (rows=525/319 width=178) @@ -1521,7 +1521,7 @@ Stage-5 Group By Operator [GBY_81] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 27 [SIMPLE_EDGE] - <-Map 26 [CONTAINS] + <-Map 26 [CONTAINS] llap Reduce Output Operator [RS_80] PartitionCols:_col1, _col0 Select Operator [SEL_73] (rows=25/25 width=175) @@ -1530,7 +1530,7 @@ Stage-5 predicate:value is not null TableScan [TS_71] (rows=25/25 width=175) Output:["key","value"] - <-Map 33 [CONTAINS] + <-Map 33 [CONTAINS] llap Reduce Output Operator [RS_80] PartitionCols:_col1, _col0 Select Operator [SEL_76] (rows=500/500 width=178) @@ -1539,20 +1539,20 @@ Stage-5 predicate:value is not null TableScan [TS_74] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 7 [CONTAINS] + <-Reducer 7 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Group By Operator [GBY_63] (rows=2749/309 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 6 [SIMPLE_EDGE] - <-Reducer 16 [CONTAINS] + <-Reducer 16 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_58] (rows=2682/1056 width=178) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_162] (rows=2682/1056 width=178) Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] - <-Map 24 [SIMPLE_EDGE] + <-Map 24 [SIMPLE_EDGE] llap SHUFFLE [RS_56] PartitionCols:_col0 Select Operator [SEL_51] (rows=500/500 width=178) @@ -1561,12 +1561,12 @@ Stage-5 predicate:key is not null TableScan [TS_49] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 15 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] llap SHUFFLE [RS_55] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_161] (rows=1658/512 width=87) Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] - <-Map 14 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] llap SHUFFLE [RS_53] PartitionCols:_col1 Select Operator [SEL_48] (rows=500/500 width=178) @@ -1575,7 +1575,7 @@ Stage-5 predicate:(key is not null and value is not null) TableScan [TS_15] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 21 [SIMPLE_EDGE] + <-Reducer 21 [SIMPLE_EDGE] llap SHUFFLE [RS_52] PartitionCols:_col1 Select Operator [SEL_45] (rows=1025/319 width=178) @@ -1583,7 +1583,7 @@ Stage-5 Group By Operator [GBY_44] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 20 [SIMPLE_EDGE] - <-Map 23 [CONTAINS] + <-Map 23 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_39] (rows=500/500 width=178) @@ -1592,7 +1592,7 @@ Stage-5 predicate:value is not null TableScan [TS_37] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 19 [CONTAINS] + <-Reducer 19 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_36] (rows=525/319 width=178) @@ -1600,7 +1600,7 @@ Stage-5 Group By Operator [GBY_35] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 18 [SIMPLE_EDGE] - <-Map 17 [CONTAINS] + <-Map 17 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_27] (rows=25/25 width=175) @@ -1609,7 +1609,7 @@ Stage-5 predicate:value is not null TableScan [TS_25] (rows=25/25 width=175) Output:["key","value"] - <-Map 22 [CONTAINS] + <-Map 22 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_30] (rows=500/500 width=178) @@ -1618,14 +1618,14 @@ Stage-5 predicate:value is not null TableScan [TS_28] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] + <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_24] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_160] (rows=67/61 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] - <-Map 14 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col0 Select Operator [SEL_17] (rows=500/500 width=178) @@ -1633,16 +1633,16 @@ Stage-5 Filter Operator [FIL_147] (rows=500/500 width=178) predicate:key is not null Please refer to the previous TableScan [TS_15] - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_159] (rows=42/52 width=86) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 Select Operator [SEL_11] (rows=525/319 width=178) @@ -1650,7 +1650,7 @@ Stage-5 Group By Operator [GBY_10] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_2] (rows=25/25 width=175) @@ -1659,7 +1659,7 @@ Stage-5 predicate:value is not null TableScan [TS_0] (rows=25/25 width=175) Output:["key","value"] - <-Map 10 [CONTAINS] + <-Map 10 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_5] (rows=500/500 width=178) @@ -1751,34 +1751,34 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 5 + Reducer 5 llap File Output Operator [FS_18] table:{"name:":"default.dest1"} Select Operator [SEL_16] (rows=309/310 width=272) Output:["_col0","_col1"] Group By Operator [GBY_15] (rows=309/310 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_14] PartitionCols:_col0 Group By Operator [GBY_11] (rows=501/310 width=272) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Select Operator [SEL_6] (rows=500/500 width=178) Output:["_col0","_col1"] TableScan [TS_5] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Select Operator [SEL_4] (rows=1/1 width=272) Output:["_col0","_col1"] Group By Operator [GBY_3] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] + <-Map 1 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] Select Operator [SEL_1] (rows=500/500 width=10) TableScan [TS_0] (rows=500/500 width=10) @@ -1830,86 +1830,28 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 llap File Output Operator [FS_8] Select Operator [SEL_7] (rows=261/4122 width=260) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_9] (rows=261/4122 width=260) Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_3] PartitionCols:key TableScan [TS_0] (rows=500/500 width=87) default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] + <-Map 3 [SIMPLE_EDGE] llap SHUFFLE [RS_4] PartitionCols:key TableScan [TS_1] (rows=25/25 width=86) default@src1,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] + <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_5] PartitionCols:key TableScan [TS_2] (rows=2000/2000 width=87) default@srcpart,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] -PREHOOK: query: SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@src -#### A masked pattern was here #### -PREHOOK: query: explain analyze -SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -PREHOOK: type: QUERY -POSTHOOK: query: explain analyze -SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -POSTHOOK: type: QUERY -Plan not optimized by CBO. - -Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) - -Stage-0 - Fetch Operator - limit:-1 - Stage-1 - Reducer 2 - File Output Operator [FS_9] - Transform Operator [SCR_8] (rows=809/1028 width=178) - command:cat - Merge Join Operator [MERGEJOIN_14] (rows=809/1028 width=178) - Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - Filter Operator [FIL_12] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_1] (rows=500/500 width=87) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - PREHOOK: query: FROM ( select key, value from ( select 'tst1' as key, cast(count(1) as string) as value, 'tst1' as value2 from src s1 @@ -1983,7 +1925,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 + Reducer 4 llap File Output Operator [FS_16] table:{"name:":"default.dest1"} Select Operator [SEL_14] (rows=309/310 width=272) @@ -1991,7 +1933,7 @@ Stage-4 Group By Operator [GBY_13] (rows=309/310 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_12] PartitionCols:_col0 Select Operator [SEL_6] (rows=500/500 width=178) @@ -2001,7 +1943,7 @@ Stage-4 Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_6] - <-Map 7 [CONTAINS] + <-Map 7 [CONTAINS] llap Reduce Output Operator [RS_12] PartitionCols:_col0 Select Operator [SEL_9] (rows=500/500 width=178) @@ -2011,14 +1953,14 @@ Stage-4 Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_9] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_12] PartitionCols:_col0 Select Operator [SEL_4] (rows=1/1 width=272) Output:["_col0","_col1"] Group By Operator [GBY_3] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] + <-Map 1 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] Select Operator [SEL_1] (rows=500/500 width=10) TableScan [TS_0] (rows=500/500 width=10) @@ -2026,7 +1968,7 @@ Stage-4 Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_4] - Reducer 5 + Reducer 5 llap File Output Operator [FS_22] table:{"name:":"default.dest2"} Select Operator [SEL_20] (rows=1001/310 width=456) @@ -2101,7 +2043,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 + Reducer 4 llap File Output Operator [FS_14] table:{"name:":"default.dest1"} Select Operator [SEL_12] (rows=309/310 width=272) @@ -2109,7 +2051,7 @@ Stage-4 Group By Operator [GBY_11] (rows=309/310 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0 Select Operator [SEL_6] (rows=500/500 width=178) @@ -2119,14 +2061,14 @@ Stage-4 Reduce Output Operator [RS_16] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_6] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0 Select Operator [SEL_4] (rows=1/1 width=272) Output:["_col0","_col1"] Group By Operator [GBY_3] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] + <-Map 1 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] Select Operator [SEL_1] (rows=500/500 width=10) TableScan [TS_0] (rows=500/500 width=10) @@ -2134,7 +2076,7 @@ Stage-4 Reduce Output Operator [RS_16] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_4] - Reducer 5 + Reducer 5 llap File Output Operator [FS_20] table:{"name:":"default.dest2"} Select Operator [SEL_18] (rows=501/310 width=456) diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out index 6b930c8a87..6f29ddaea1 100644 --- a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out +++ b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out @@ -60,8 +60,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs Map 8 Map Operator Tree: TableScan @@ -78,10 +76,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs Reducer 2 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -141,7 +136,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable Reducer 7 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -163,7 +157,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Reducer 9 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -237,215 +230,6 @@ POSTHOOK: Input: default@tmptable tst1 500 tst2 500 tst3 500 -PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst3' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst2' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -tst1 1 -tst2 1 -tst3 1 PREHOOK: query: create table tmptable12(key string, value int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -504,8 +288,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs Map 7 Map Operator Tree: TableScan @@ -522,8 +304,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs Map 9 Map Operator Tree: TableScan @@ -540,10 +320,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs Reducer 10 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -565,7 +342,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -625,7 +401,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable12 Reducer 8 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -744,8 +519,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs Map 4 Map Operator Tree: TableScan @@ -765,8 +538,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -1110,195 +881,60 @@ POSTHOOK: Input: default@src 83 val_83 86 val_86 98 val_98 +PREHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST1 +POSTHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST1 +PREHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST2 +POSTHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST2 PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value PREHOOK: type: QUERY POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-2 Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Map 6 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -#### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -#### A masked pattern was here #### - 7 -128 1 -146 1 -150 1 -213 1 -224 1 -238 1 -255 1 -273 1 -278 1 -311 1 -369 1 -401 1 -406 1 -66 1 -98 1 -tst1 1 -PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 7 <- Union 3 (CONTAINS) - Map 8 <- Union 5 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -1310,52 +946,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Map 6 Map Operator Tree: TableScan alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ + key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1370,12 +980,12 @@ STAGE PLANS: keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ + key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -1383,9645 +993,1721 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 125 Data size: 35000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Statistics: Num rows: 125 Data size: 35000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), UDFToString(_col2) (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + Reducer 5 Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + aggregations: count(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 12000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 125 Data size: 34000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 34000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Union 3 Vertex: Union 3 - Union 5 - Vertex: Union 5 + + Stage: Stage-3 + Dependency Collection Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 -PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -#### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -#### A masked pattern was here #### - 7 -128 1 -146 1 -150 1 -213 1 -224 1 -238 1 + Stage: Stage-4 + Stats Work + Basic Stats Work: + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1 +PREHOOK: Output: default@dest2 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1 +POSTHOOK: Output: default@dest2 +POSTHOOK: Lineage: dest1.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST1.* FROM DEST1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST1.* FROM DEST1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 273 1 +274 1 +275 1 +277 1 278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 401 1 +402 1 +403 1 +404 1 406 1 -66 1 -98 1 -tst1 1 -PREHOOK: query: EXPLAIN -SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +tst1 1 +PREHOOK: query: SELECT DEST2.* FROM DEST2 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez +PREHOOK: Input: default@dest2 #### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 51 <- Union 2 (CONTAINS) - Map 52 <- Union 4 (CONTAINS) - Map 53 <- Union 6 (CONTAINS) - Map 54 <- Union 8 (CONTAINS) - Map 55 <- Union 10 (CONTAINS) - Map 56 <- Union 12 (CONTAINS) - Map 57 <- Union 14 (CONTAINS) - Map 58 <- Union 16 (CONTAINS) - Map 59 <- Union 18 (CONTAINS) - Map 60 <- Union 20 (CONTAINS) - Map 61 <- Union 22 (CONTAINS) - Map 62 <- Union 24 (CONTAINS) - Map 63 <- Union 26 (CONTAINS) - Map 64 <- Union 28 (CONTAINS) - Map 65 <- Union 30 (CONTAINS) - Map 66 <- Union 32 (CONTAINS) - Map 67 <- Union 34 (CONTAINS) - Map 68 <- Union 36 (CONTAINS) - Map 69 <- Union 38 (CONTAINS) - Map 70 <- Union 40 (CONTAINS) - Map 71 <- Union 42 (CONTAINS) - Map 72 <- Union 44 (CONTAINS) - Map 73 <- Union 46 (CONTAINS) - Map 74 <- Union 48 (CONTAINS) - Reducer 11 <- Union 10 (SIMPLE_EDGE), Union 12 (CONTAINS) - Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) - Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) - Reducer 17 <- Union 16 (SIMPLE_EDGE), Union 18 (CONTAINS) - Reducer 19 <- Union 18 (SIMPLE_EDGE), Union 20 (CONTAINS) - Reducer 21 <- Union 20 (SIMPLE_EDGE), Union 22 (CONTAINS) - Reducer 23 <- Union 22 (SIMPLE_EDGE), Union 24 (CONTAINS) - Reducer 25 <- Union 24 (SIMPLE_EDGE), Union 26 (CONTAINS) - Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) - Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 31 <- Union 30 (SIMPLE_EDGE), Union 32 (CONTAINS) - Reducer 33 <- Union 32 (SIMPLE_EDGE), Union 34 (CONTAINS) - Reducer 35 <- Union 34 (SIMPLE_EDGE), Union 36 (CONTAINS) - Reducer 37 <- Union 36 (SIMPLE_EDGE), Union 38 (CONTAINS) - Reducer 39 <- Union 38 (SIMPLE_EDGE), Union 40 (CONTAINS) - Reducer 41 <- Union 40 (SIMPLE_EDGE), Union 42 (CONTAINS) - Reducer 43 <- Union 42 (SIMPLE_EDGE), Union 44 (CONTAINS) - Reducer 45 <- Union 44 (SIMPLE_EDGE), Union 46 (CONTAINS) - Reducer 47 <- Union 46 (SIMPLE_EDGE), Union 48 (CONTAINS) - Reducer 49 <- Union 48 (SIMPLE_EDGE) - Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 50 <- Reducer 49 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 9 <- Union 10 (CONTAINS), Union 8 (SIMPLE_EDGE) +POSTHOOK: query: SELECT DEST2.* FROM DEST2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest2 #### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 51 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 52 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 53 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 54 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 55 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 56 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 57 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 58 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 59 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 60 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 61 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 62 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 63 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 64 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 65 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 66 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 67 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 68 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 69 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 70 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 71 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 72 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 73 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 74 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 15 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 25 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 27 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 29 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 31 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 33 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 35 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 37 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 39 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 41 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 43 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 45 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 47 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 49 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 50 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union 10 - Vertex: Union 10 - Union 12 - Vertex: Union 12 - Union 14 - Vertex: Union 14 - Union 16 - Vertex: Union 16 - Union 18 - Vertex: Union 18 - Union 2 - Vertex: Union 2 - Union 20 - Vertex: Union 20 - Union 22 - Vertex: Union 22 - Union 24 - Vertex: Union 24 - Union 26 - Vertex: Union 26 - Union 28 - Vertex: Union 28 - Union 30 - Vertex: Union 30 - Union 32 - Vertex: Union 32 - Union 34 - Vertex: Union 34 - Union 36 - Vertex: Union 36 - Union 38 - Vertex: Union 38 - Union 4 - Vertex: Union 4 - Union 40 - Vertex: Union 40 - Union 42 - Vertex: Union 42 - Union 44 - Vertex: Union 44 - Union 46 - Vertex: Union 46 - Union 48 - Vertex: Union 48 - Union 6 - Vertex: Union 6 - Union 8 - Vertex: Union 8 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -309 -PREHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@DEST1 -POSTHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@DEST1 -PREHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@DEST2 -POSTHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@DEST2 -PREHOOK: query: explain -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key -INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value -PREHOOK: type: QUERY -POSTHOOK: query: explain -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key -INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-2 - Tez -#### A masked pattern was here #### - Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 125 Data size: 35000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 125 Data size: 35000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), UDFToString(_col2) (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2 - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(DISTINCT KEY._col1:0._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 12000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToString(_col1) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 34000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 125 Data size: 34000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 - Union 3 - Vertex: Union 3 - - Stage: Stage-3 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2 - - Stage: Stage-5 - Stats Work - Basic Stats Work: - -PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key -INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1 -PREHOOK: Output: default@dest2 -POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key -INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1 -POSTHOOK: Output: default@dest2 -POSTHOOK: Lineage: dest1.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dest1.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest2.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dest2.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest2.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT DEST1.* FROM DEST1 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT DEST1.* FROM DEST1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1 -#### A masked pattern was here #### -0 1 -10 1 -100 1 -103 1 -104 1 -105 1 -11 1 -111 1 -113 1 -114 1 -116 1 -118 1 -119 1 -12 1 -120 1 -125 1 -126 1 -128 1 -129 1 -131 1 -133 1 -134 1 -136 1 -137 1 -138 1 -143 1 -145 1 -146 1 -149 1 -15 1 -150 1 -152 1 -153 1 -155 1 -156 1 -157 1 -158 1 -160 1 -162 1 -163 1 -164 1 -165 1 -166 1 -167 1 -168 1 -169 1 -17 1 -170 1 -172 1 -174 1 -175 1 -176 1 -177 1 -178 1 -179 1 -18 1 -180 1 -181 1 -183 1 -186 1 -187 1 -189 1 -19 1 -190 1 -191 1 -192 1 -193 1 -194 1 -195 1 -196 1 -197 1 -199 1 -2 1 -20 1 -200 1 -201 1 -202 1 -203 1 -205 1 -207 1 -208 1 -209 1 -213 1 -214 1 -216 1 -217 1 -218 1 -219 1 -221 1 -222 1 -223 1 -224 1 -226 1 -228 1 -229 1 -230 1 -233 1 -235 1 -237 1 -238 1 -239 1 -24 1 -241 1 -242 1 -244 1 -247 1 -248 1 -249 1 -252 1 -255 1 -256 1 -257 1 -258 1 -26 1 -260 1 -262 1 -263 1 -265 1 -266 1 -27 1 -272 1 -273 1 -274 1 -275 1 -277 1 -278 1 -28 1 -280 1 -281 1 -282 1 -283 1 -284 1 -285 1 -286 1 -287 1 -288 1 -289 1 -291 1 -292 1 -296 1 -298 1 -30 1 -302 1 -305 1 -306 1 -307 1 -308 1 -309 1 -310 1 -311 1 -315 1 -316 1 -317 1 -318 1 -321 1 -322 1 -323 1 -325 1 -327 1 -33 1 -331 1 -332 1 -333 1 -335 1 -336 1 -338 1 -339 1 -34 1 -341 1 -342 1 -344 1 -345 1 -348 1 -35 1 -351 1 -353 1 -356 1 -360 1 -362 1 -364 1 -365 1 -366 1 -367 1 -368 1 -369 1 -37 1 -373 1 -374 1 -375 1 -377 1 -378 1 -379 1 -382 1 -384 1 -386 1 -389 1 -392 1 -393 1 -394 1 -395 1 -396 1 -397 1 -399 1 -4 1 -400 1 -401 1 -402 1 -403 1 -404 1 -406 1 -407 1 -409 1 -41 1 -411 1 -413 1 -414 1 -417 1 -418 1 -419 1 -42 1 -421 1 -424 1 -427 1 -429 1 -43 1 -430 1 -431 1 -432 1 -435 1 -436 1 -437 1 -438 1 -439 1 -44 1 -443 1 -444 1 -446 1 -448 1 -449 1 -452 1 -453 1 -454 1 -455 1 -457 1 -458 1 -459 1 -460 1 -462 1 -463 1 -466 1 -467 1 -468 1 -469 1 -47 1 -470 1 -472 1 -475 1 -477 1 -478 1 -479 1 -480 1 -481 1 -482 1 -483 1 -484 1 -485 1 -487 1 -489 1 -490 1 -491 1 -492 1 -493 1 -494 1 -495 1 -496 1 -497 1 -498 1 -5 1 -51 1 -53 1 -54 1 -57 1 -58 1 -64 1 -65 1 -66 1 -67 1 -69 1 -70 1 -72 1 -74 1 -76 1 -77 1 -78 1 -8 1 -80 1 -82 1 -83 1 -84 1 -85 1 -86 1 -87 1 -9 1 -90 1 -92 1 -95 1 -96 1 -97 1 -98 1 -tst1 1 -PREHOOK: query: SELECT DEST2.* FROM DEST2 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT DEST2.* FROM DEST2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest2 -#### A masked pattern was here #### -0 val_0 1 -10 val_10 1 -100 val_100 1 -103 val_103 1 -104 val_104 1 -105 val_105 1 -11 val_11 1 -111 val_111 1 -113 val_113 1 -114 val_114 1 -116 val_116 1 -118 val_118 1 -119 val_119 1 -12 val_12 1 -120 val_120 1 -125 val_125 1 -126 val_126 1 -128 val_128 1 -129 val_129 1 -131 val_131 1 -133 val_133 1 -134 val_134 1 -136 val_136 1 -137 val_137 1 -138 val_138 1 -143 val_143 1 -145 val_145 1 -146 val_146 1 -149 val_149 1 -15 val_15 1 -150 val_150 1 -152 val_152 1 -153 val_153 1 -155 val_155 1 -156 val_156 1 -157 val_157 1 -158 val_158 1 -160 val_160 1 -162 val_162 1 -163 val_163 1 -164 val_164 1 -165 val_165 1 -166 val_166 1 -167 val_167 1 -168 val_168 1 -169 val_169 1 -17 val_17 1 -170 val_170 1 -172 val_172 1 -174 val_174 1 -175 val_175 1 -176 val_176 1 -177 val_177 1 -178 val_178 1 -179 val_179 1 -18 val_18 1 -180 val_180 1 -181 val_181 1 -183 val_183 1 -186 val_186 1 -187 val_187 1 -189 val_189 1 -19 val_19 1 -190 val_190 1 -191 val_191 1 -192 val_192 1 -193 val_193 1 -194 val_194 1 -195 val_195 1 -196 val_196 1 -197 val_197 1 -199 val_199 1 -2 val_2 1 -20 val_20 1 -200 val_200 1 -201 val_201 1 -202 val_202 1 -203 val_203 1 -205 val_205 1 -207 val_207 1 -208 val_208 1 -209 val_209 1 -213 val_213 1 -214 val_214 1 -216 val_216 1 -217 val_217 1 -218 val_218 1 -219 val_219 1 -221 val_221 1 -222 val_222 1 -223 val_223 1 -224 val_224 1 -226 val_226 1 -228 val_228 1 -229 val_229 1 -230 val_230 1 -233 val_233 1 -235 val_235 1 -237 val_237 1 -238 val_238 1 -239 val_239 1 -24 val_24 1 -241 val_241 1 -242 val_242 1 -244 val_244 1 -247 val_247 1 -248 val_248 1 -249 val_249 1 -252 val_252 1 -255 val_255 1 -256 val_256 1 -257 val_257 1 -258 val_258 1 -26 val_26 1 -260 val_260 1 -262 val_262 1 -263 val_263 1 -265 val_265 1 -266 val_266 1 -27 val_27 1 -272 val_272 1 -273 val_273 1 -274 val_274 1 -275 val_275 1 -277 val_277 1 -278 val_278 1 -28 val_28 1 -280 val_280 1 -281 val_281 1 -282 val_282 1 -283 val_283 1 -284 val_284 1 -285 val_285 1 -286 val_286 1 -287 val_287 1 -288 val_288 1 -289 val_289 1 -291 val_291 1 -292 val_292 1 -296 val_296 1 -298 val_298 1 -30 val_30 1 -302 val_302 1 -305 val_305 1 -306 val_306 1 -307 val_307 1 -308 val_308 1 -309 val_309 1 -310 val_310 1 -311 val_311 1 -315 val_315 1 -316 val_316 1 -317 val_317 1 -318 val_318 1 -321 val_321 1 -322 val_322 1 -323 val_323 1 -325 val_325 1 -327 val_327 1 -33 val_33 1 -331 val_331 1 -332 val_332 1 -333 val_333 1 -335 val_335 1 -336 val_336 1 -338 val_338 1 -339 val_339 1 -34 val_34 1 -341 val_341 1 -342 val_342 1 -344 val_344 1 -345 val_345 1 -348 val_348 1 -35 val_35 1 -351 val_351 1 -353 val_353 1 -356 val_356 1 -360 val_360 1 -362 val_362 1 -364 val_364 1 -365 val_365 1 -366 val_366 1 -367 val_367 1 -368 val_368 1 -369 val_369 1 -37 val_37 1 -373 val_373 1 -374 val_374 1 -375 val_375 1 -377 val_377 1 -378 val_378 1 -379 val_379 1 -382 val_382 1 -384 val_384 1 -386 val_386 1 -389 val_389 1 -392 val_392 1 -393 val_393 1 -394 val_394 1 -395 val_395 1 -396 val_396 1 -397 val_397 1 -399 val_399 1 -4 val_4 1 -400 val_400 1 -401 val_401 1 -402 val_402 1 -403 val_403 1 -404 val_404 1 -406 val_406 1 -407 val_407 1 -409 val_409 1 -41 val_41 1 -411 val_411 1 -413 val_413 1 -414 val_414 1 -417 val_417 1 -418 val_418 1 -419 val_419 1 -42 val_42 1 -421 val_421 1 -424 val_424 1 -427 val_427 1 -429 val_429 1 -43 val_43 1 -430 val_430 1 -431 val_431 1 -432 val_432 1 -435 val_435 1 -436 val_436 1 -437 val_437 1 -438 val_438 1 -439 val_439 1 -44 val_44 1 -443 val_443 1 -444 val_444 1 -446 val_446 1 -448 val_448 1 -449 val_449 1 -452 val_452 1 -453 val_453 1 -454 val_454 1 -455 val_455 1 -457 val_457 1 -458 val_458 1 -459 val_459 1 -460 val_460 1 -462 val_462 1 -463 val_463 1 -466 val_466 1 -467 val_467 1 -468 val_468 1 -469 val_469 1 -47 val_47 1 -470 val_470 1 -472 val_472 1 -475 val_475 1 -477 val_477 1 -478 val_478 1 -479 val_479 1 -480 val_480 1 -481 val_481 1 -482 val_482 1 -483 val_483 1 -484 val_484 1 -485 val_485 1 -487 val_487 1 -489 val_489 1 -490 val_490 1 -491 val_491 1 -492 val_492 1 -493 val_493 1 -494 val_494 1 -495 val_495 1 -496 val_496 1 -497 val_497 1 -498 val_498 1 -5 val_5 1 -51 val_51 1 -53 val_53 1 -54 val_54 1 -57 val_57 1 -58 val_58 1 -64 val_64 1 -65 val_65 1 -66 val_66 1 -67 val_67 1 -69 val_69 1 -70 val_70 1 -72 val_72 1 -74 val_74 1 -76 val_76 1 -77 val_77 1 -78 val_78 1 -8 val_8 1 -80 val_80 1 -82 val_82 1 -83 val_83 1 -84 val_84 1 -85 val_85 1 -86 val_86 1 -87 val_87 1 -9 val_9 1 -90 val_90 1 -92 val_92 1 -95 val_95 1 -96 val_96 1 -97 val_97 1 -98 val_98 1 -tst1 500 1 -PREHOOK: query: CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@DEST118 -POSTHOOK: query: CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@DEST118 -PREHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@DEST218 -POSTHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@DEST218 -PREHOOK: query: explain -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value -INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value -PREHOOK: type: QUERY -POSTHOOK: query: explain -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value -INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-2 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest118 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest218 - Union 3 - Vertex: Union 3 - - Stage: Stage-3 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest118 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest218 - - Stage: Stage-5 - Stats Work - Basic Stats Work: - -PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value -INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest118 -PREHOOK: Output: default@dest218 -POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value -INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest118 -POSTHOOK: Output: default@dest218 -POSTHOOK: Lineage: dest118.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dest118.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest218.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dest218.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest218.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value -PREHOOK: type: QUERY -PREHOOK: Input: default@dest118 -#### A masked pattern was here #### -POSTHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest118 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -103 val_103 -104 val_104 -105 val_105 -11 val_11 -111 val_111 -113 val_113 -114 val_114 -116 val_116 -118 val_118 -119 val_119 -12 val_12 -120 val_120 -125 val_125 -126 val_126 -128 val_128 -129 val_129 -131 val_131 -133 val_133 -134 val_134 -136 val_136 -137 val_137 -138 val_138 -143 val_143 -145 val_145 -146 val_146 -149 val_149 -15 val_15 -150 val_150 -152 val_152 -153 val_153 -155 val_155 -156 val_156 -157 val_157 -158 val_158 -160 val_160 -162 val_162 -163 val_163 -164 val_164 -165 val_165 -166 val_166 -167 val_167 -168 val_168 -169 val_169 -17 val_17 -170 val_170 -172 val_172 -174 val_174 -175 val_175 -176 val_176 -177 val_177 -178 val_178 -179 val_179 -18 val_18 -180 val_180 -181 val_181 -183 val_183 -186 val_186 -187 val_187 -189 val_189 -19 val_19 -190 val_190 -191 val_191 -192 val_192 -193 val_193 -194 val_194 -195 val_195 -196 val_196 -197 val_197 -199 val_199 -2 val_2 -20 val_20 -200 val_200 -201 val_201 -202 val_202 -203 val_203 -205 val_205 -207 val_207 -208 val_208 -209 val_209 -213 val_213 -214 val_214 -216 val_216 -217 val_217 -218 val_218 -219 val_219 -221 val_221 -222 val_222 -223 val_223 -224 val_224 -226 val_226 -228 val_228 -229 val_229 -230 val_230 -233 val_233 -235 val_235 -237 val_237 -238 val_238 -239 val_239 -24 val_24 -241 val_241 -242 val_242 -244 val_244 -247 val_247 -248 val_248 -249 val_249 -252 val_252 -255 val_255 -256 val_256 -257 val_257 -258 val_258 -26 val_26 -260 val_260 -262 val_262 -263 val_263 -265 val_265 -266 val_266 -27 val_27 -272 val_272 -273 val_273 -274 val_274 -275 val_275 -277 val_277 -278 val_278 -28 val_28 -280 val_280 -281 val_281 -282 val_282 -283 val_283 -284 val_284 -285 val_285 -286 val_286 -287 val_287 -288 val_288 -289 val_289 -291 val_291 -292 val_292 -296 val_296 -298 val_298 -30 val_30 -302 val_302 -305 val_305 -306 val_306 -307 val_307 -308 val_308 -309 val_309 -310 val_310 -311 val_311 -315 val_315 -316 val_316 -317 val_317 -318 val_318 -321 val_321 -322 val_322 -323 val_323 -325 val_325 -327 val_327 -33 val_33 -331 val_331 -332 val_332 -333 val_333 -335 val_335 -336 val_336 -338 val_338 -339 val_339 -34 val_34 -341 val_341 -342 val_342 -344 val_344 -345 val_345 -348 val_348 -35 val_35 -351 val_351 -353 val_353 -356 val_356 -360 val_360 -362 val_362 -364 val_364 -365 val_365 -366 val_366 -367 val_367 -368 val_368 -369 val_369 -37 val_37 -373 val_373 -374 val_374 -375 val_375 -377 val_377 -378 val_378 -379 val_379 -382 val_382 -384 val_384 -386 val_386 -389 val_389 -392 val_392 -393 val_393 -394 val_394 -395 val_395 -396 val_396 -397 val_397 -399 val_399 -4 val_4 -400 val_400 -401 val_401 -402 val_402 -403 val_403 -404 val_404 -406 val_406 -407 val_407 -409 val_409 -41 val_41 -411 val_411 -413 val_413 -414 val_414 -417 val_417 -418 val_418 -419 val_419 -42 val_42 -421 val_421 -424 val_424 -427 val_427 -429 val_429 -43 val_43 -430 val_430 -431 val_431 -432 val_432 -435 val_435 -436 val_436 -437 val_437 -438 val_438 -439 val_439 -44 val_44 -443 val_443 -444 val_444 -446 val_446 -448 val_448 -449 val_449 -452 val_452 -453 val_453 -454 val_454 -455 val_455 -457 val_457 -458 val_458 -459 val_459 -460 val_460 -462 val_462 -463 val_463 -466 val_466 -467 val_467 -468 val_468 -469 val_469 -47 val_47 -470 val_470 -472 val_472 -475 val_475 -477 val_477 -478 val_478 -479 val_479 -480 val_480 -481 val_481 -482 val_482 -483 val_483 -484 val_484 -485 val_485 -487 val_487 -489 val_489 -490 val_490 -491 val_491 -492 val_492 -493 val_493 -494 val_494 -495 val_495 -496 val_496 -497 val_497 -498 val_498 -5 val_5 -51 val_51 -53 val_53 -54 val_54 -57 val_57 -58 val_58 -64 val_64 -65 val_65 -66 val_66 -67 val_67 -69 val_69 -70 val_70 -72 val_72 -74 val_74 -76 val_76 -77 val_77 -78 val_78 -8 val_8 -80 val_80 -82 val_82 -83 val_83 -84 val_84 -85 val_85 -86 val_86 -87 val_87 -9 val_9 -90 val_90 -92 val_92 -95 val_95 -96 val_96 -97 val_97 -98 val_98 -tst1 500 -PREHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest218 -#### A masked pattern was here #### -POSTHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest218 -#### A masked pattern was here #### -0 val_0 val_0 -10 val_10 val_10 -100 val_100 val_100 -103 val_103 val_103 -104 val_104 val_104 -105 val_105 val_105 -11 val_11 val_11 -111 val_111 val_111 -113 val_113 val_113 -114 val_114 val_114 -116 val_116 val_116 -118 val_118 val_118 -119 val_119 val_119 -12 val_12 val_12 -120 val_120 val_120 -125 val_125 val_125 -126 val_126 val_126 -128 val_128 val_128 -129 val_129 val_129 -131 val_131 val_131 -133 val_133 val_133 -134 val_134 val_134 -136 val_136 val_136 -137 val_137 val_137 -138 val_138 val_138 -143 val_143 val_143 -145 val_145 val_145 -146 val_146 val_146 -149 val_149 val_149 -15 val_15 val_15 -150 val_150 val_150 -152 val_152 val_152 -153 val_153 val_153 -155 val_155 val_155 -156 val_156 val_156 -157 val_157 val_157 -158 val_158 val_158 -160 val_160 val_160 -162 val_162 val_162 -163 val_163 val_163 -164 val_164 val_164 -165 val_165 val_165 -166 val_166 val_166 -167 val_167 val_167 -168 val_168 val_168 -169 val_169 val_169 -17 val_17 val_17 -170 val_170 val_170 -172 val_172 val_172 -174 val_174 val_174 -175 val_175 val_175 -176 val_176 val_176 -177 val_177 val_177 -178 val_178 val_178 -179 val_179 val_179 -18 val_18 val_18 -180 val_180 val_180 -181 val_181 val_181 -183 val_183 val_183 -186 val_186 val_186 -187 val_187 val_187 -189 val_189 val_189 -19 val_19 val_19 -190 val_190 val_190 -191 val_191 val_191 -192 val_192 val_192 -193 val_193 val_193 -194 val_194 val_194 -195 val_195 val_195 -196 val_196 val_196 -197 val_197 val_197 -199 val_199 val_199 -2 val_2 val_2 -20 val_20 val_20 -200 val_200 val_200 -201 val_201 val_201 -202 val_202 val_202 -203 val_203 val_203 -205 val_205 val_205 -207 val_207 val_207 -208 val_208 val_208 -209 val_209 val_209 -213 val_213 val_213 -214 val_214 val_214 -216 val_216 val_216 -217 val_217 val_217 -218 val_218 val_218 -219 val_219 val_219 -221 val_221 val_221 -222 val_222 val_222 -223 val_223 val_223 -224 val_224 val_224 -226 val_226 val_226 -228 val_228 val_228 -229 val_229 val_229 -230 val_230 val_230 -233 val_233 val_233 -235 val_235 val_235 -237 val_237 val_237 -238 val_238 val_238 -239 val_239 val_239 -24 val_24 val_24 -241 val_241 val_241 -242 val_242 val_242 -244 val_244 val_244 -247 val_247 val_247 -248 val_248 val_248 -249 val_249 val_249 -252 val_252 val_252 -255 val_255 val_255 -256 val_256 val_256 -257 val_257 val_257 -258 val_258 val_258 -26 val_26 val_26 -260 val_260 val_260 -262 val_262 val_262 -263 val_263 val_263 -265 val_265 val_265 -266 val_266 val_266 -27 val_27 val_27 -272 val_272 val_272 -273 val_273 val_273 -274 val_274 val_274 -275 val_275 val_275 -277 val_277 val_277 -278 val_278 val_278 -28 val_28 val_28 -280 val_280 val_280 -281 val_281 val_281 -282 val_282 val_282 -283 val_283 val_283 -284 val_284 val_284 -285 val_285 val_285 -286 val_286 val_286 -287 val_287 val_287 -288 val_288 val_288 -289 val_289 val_289 -291 val_291 val_291 -292 val_292 val_292 -296 val_296 val_296 -298 val_298 val_298 -30 val_30 val_30 -302 val_302 val_302 -305 val_305 val_305 -306 val_306 val_306 -307 val_307 val_307 -308 val_308 val_308 -309 val_309 val_309 -310 val_310 val_310 -311 val_311 val_311 -315 val_315 val_315 -316 val_316 val_316 -317 val_317 val_317 -318 val_318 val_318 -321 val_321 val_321 -322 val_322 val_322 -323 val_323 val_323 -325 val_325 val_325 -327 val_327 val_327 -33 val_33 val_33 -331 val_331 val_331 -332 val_332 val_332 -333 val_333 val_333 -335 val_335 val_335 -336 val_336 val_336 -338 val_338 val_338 -339 val_339 val_339 -34 val_34 val_34 -341 val_341 val_341 -342 val_342 val_342 -344 val_344 val_344 -345 val_345 val_345 -348 val_348 val_348 -35 val_35 val_35 -351 val_351 val_351 -353 val_353 val_353 -356 val_356 val_356 -360 val_360 val_360 -362 val_362 val_362 -364 val_364 val_364 -365 val_365 val_365 -366 val_366 val_366 -367 val_367 val_367 -368 val_368 val_368 -369 val_369 val_369 -37 val_37 val_37 -373 val_373 val_373 -374 val_374 val_374 -375 val_375 val_375 -377 val_377 val_377 -378 val_378 val_378 -379 val_379 val_379 -382 val_382 val_382 -384 val_384 val_384 -386 val_386 val_386 -389 val_389 val_389 -392 val_392 val_392 -393 val_393 val_393 -394 val_394 val_394 -395 val_395 val_395 -396 val_396 val_396 -397 val_397 val_397 -399 val_399 val_399 -4 val_4 val_4 -400 val_400 val_400 -401 val_401 val_401 -402 val_402 val_402 -403 val_403 val_403 -404 val_404 val_404 -406 val_406 val_406 -407 val_407 val_407 -409 val_409 val_409 -41 val_41 val_41 -411 val_411 val_411 -413 val_413 val_413 -414 val_414 val_414 -417 val_417 val_417 -418 val_418 val_418 -419 val_419 val_419 -42 val_42 val_42 -421 val_421 val_421 -424 val_424 val_424 -427 val_427 val_427 -429 val_429 val_429 -43 val_43 val_43 -430 val_430 val_430 -431 val_431 val_431 -432 val_432 val_432 -435 val_435 val_435 -436 val_436 val_436 -437 val_437 val_437 -438 val_438 val_438 -439 val_439 val_439 -44 val_44 val_44 -443 val_443 val_443 -444 val_444 val_444 -446 val_446 val_446 -448 val_448 val_448 -449 val_449 val_449 -452 val_452 val_452 -453 val_453 val_453 -454 val_454 val_454 -455 val_455 val_455 -457 val_457 val_457 -458 val_458 val_458 -459 val_459 val_459 -460 val_460 val_460 -462 val_462 val_462 -463 val_463 val_463 -466 val_466 val_466 -467 val_467 val_467 -468 val_468 val_468 -469 val_469 val_469 -47 val_47 val_47 -470 val_470 val_470 -472 val_472 val_472 -475 val_475 val_475 -477 val_477 val_477 -478 val_478 val_478 -479 val_479 val_479 -480 val_480 val_480 -481 val_481 val_481 -482 val_482 val_482 -483 val_483 val_483 -484 val_484 val_484 -485 val_485 val_485 -487 val_487 val_487 -489 val_489 val_489 -490 val_490 val_490 -491 val_491 val_491 -492 val_492 val_492 -493 val_493 val_493 -494 val_494 val_494 -495 val_495 val_495 -496 val_496 val_496 -497 val_497 val_497 -498 val_498 val_498 -5 val_5 val_5 -51 val_51 val_51 -53 val_53 val_53 -54 val_54 val_54 -57 val_57 val_57 -58 val_58 val_58 -64 val_64 val_64 -65 val_65 val_65 -66 val_66 val_66 -67 val_67 val_67 -69 val_69 val_69 -70 val_70 val_70 -72 val_72 val_72 -74 val_74 val_74 -76 val_76 val_76 -77 val_77 val_77 -78 val_78 val_78 -8 val_8 val_8 -80 val_80 val_80 -82 val_82 val_82 -83 val_83 val_83 -84 val_84 val_84 -85 val_85 val_85 -86 val_86 val_86 -87 val_87 val_87 -9 val_9 val_9 -90 val_90 val_90 -92 val_92 val_92 -95 val_95 val_95 -96 val_96 val_96 -97 val_97 val_97 -98 val_98 val_98 -tst1 500 500 -PREHOOK: query: CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@DEST119 -POSTHOOK: query: CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@DEST119 -PREHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@DEST219 -POSTHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@DEST219 -PREHOOK: query: explain -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key -INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value -PREHOOK: type: QUERY -POSTHOOK: query: explain -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key -INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-2 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col1) - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToString(_col1) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest119 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest219 - Union 3 - Vertex: Union 3 - - Stage: Stage-3 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest119 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest219 - - Stage: Stage-5 - Stats Work - Basic Stats Work: - -PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key -INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest119 -PREHOOK: Output: default@dest219 -POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key -INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest119 -POSTHOOK: Output: default@dest219 -POSTHOOK: Lineage: dest119.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dest119.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest219.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dest219.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest219.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value -PREHOOK: type: QUERY -PREHOOK: Input: default@dest119 -#### A masked pattern was here #### -POSTHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest119 -#### A masked pattern was here #### -0 1 -10 1 -100 1 -103 1 -104 1 -105 1 -11 1 -111 1 -113 1 -114 1 -116 1 -118 1 -119 1 -12 1 -120 1 -125 1 -126 1 -128 1 -129 1 -131 1 -133 1 -134 1 -136 1 -137 1 -138 1 -143 1 -145 1 -146 1 -149 1 -15 1 -150 1 -152 1 -153 1 -155 1 -156 1 -157 1 -158 1 -160 1 -162 1 -163 1 -164 1 -165 1 -166 1 -167 1 -168 1 -169 1 -17 1 -170 1 -172 1 -174 1 -175 1 -176 1 -177 1 -178 1 -179 1 -18 1 -180 1 -181 1 -183 1 -186 1 -187 1 -189 1 -19 1 -190 1 -191 1 -192 1 -193 1 -194 1 -195 1 -196 1 -197 1 -199 1 -2 1 -20 1 -200 1 -201 1 -202 1 -203 1 -205 1 -207 1 -208 1 -209 1 -213 1 -214 1 -216 1 -217 1 -218 1 -219 1 -221 1 -222 1 -223 1 -224 1 -226 1 -228 1 -229 1 -230 1 -233 1 -235 1 -237 1 -238 1 -239 1 -24 1 -241 1 -242 1 -244 1 -247 1 -248 1 -249 1 -252 1 -255 1 -256 1 -257 1 -258 1 -26 1 -260 1 -262 1 -263 1 -265 1 -266 1 -27 1 -272 1 -273 1 -274 1 -275 1 -277 1 -278 1 -28 1 -280 1 -281 1 -282 1 -283 1 -284 1 -285 1 -286 1 -287 1 -288 1 -289 1 -291 1 -292 1 -296 1 -298 1 -30 1 -302 1 -305 1 -306 1 -307 1 -308 1 -309 1 -310 1 -311 1 -315 1 -316 1 -317 1 -318 1 -321 1 -322 1 -323 1 -325 1 -327 1 -33 1 -331 1 -332 1 -333 1 -335 1 -336 1 -338 1 -339 1 -34 1 -341 1 -342 1 -344 1 -345 1 -348 1 -35 1 -351 1 -353 1 -356 1 -360 1 -362 1 -364 1 -365 1 -366 1 -367 1 -368 1 -369 1 -37 1 -373 1 -374 1 -375 1 -377 1 -378 1 -379 1 -382 1 -384 1 -386 1 -389 1 -392 1 -393 1 -394 1 -395 1 -396 1 -397 1 -399 1 -4 1 -400 1 -401 1 -402 1 -403 1 -404 1 -406 1 -407 1 -409 1 -41 1 -411 1 -413 1 -414 1 -417 1 -418 1 -419 1 -42 1 -421 1 -424 1 -427 1 -429 1 -43 1 -430 1 -431 1 -432 1 -435 1 -436 1 -437 1 -438 1 -439 1 -44 1 -443 1 -444 1 -446 1 -448 1 -449 1 -452 1 -453 1 -454 1 -455 1 -457 1 -458 1 -459 1 -460 1 -462 1 -463 1 -466 1 -467 1 -468 1 -469 1 -47 1 -470 1 -472 1 -475 1 -477 1 -478 1 -479 1 -480 1 -481 1 -482 1 -483 1 -484 1 -485 1 -487 1 -489 1 -490 1 -491 1 -492 1 -493 1 -494 1 -495 1 -496 1 -497 1 -498 1 -5 1 -51 1 -53 1 -54 1 -57 1 -58 1 -64 1 -65 1 -66 1 -67 1 -69 1 -70 1 -72 1 -74 1 -76 1 -77 1 -78 1 -8 1 -80 1 -82 1 -83 1 -84 1 -85 1 -86 1 -87 1 -9 1 -90 1 -92 1 -95 1 -96 1 -97 1 -98 1 -tst1 1 -PREHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest219 -#### A masked pattern was here #### -POSTHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest219 -#### A masked pattern was here #### -0 val_0 val_0 -10 val_10 val_10 -100 val_100 val_100 -103 val_103 val_103 -104 val_104 val_104 -105 val_105 val_105 -11 val_11 val_11 -111 val_111 val_111 -113 val_113 val_113 -114 val_114 val_114 -116 val_116 val_116 -118 val_118 val_118 -119 val_119 val_119 -12 val_12 val_12 -120 val_120 val_120 -125 val_125 val_125 -126 val_126 val_126 -128 val_128 val_128 -129 val_129 val_129 -131 val_131 val_131 -133 val_133 val_133 -134 val_134 val_134 -136 val_136 val_136 -137 val_137 val_137 -138 val_138 val_138 -143 val_143 val_143 -145 val_145 val_145 -146 val_146 val_146 -149 val_149 val_149 -15 val_15 val_15 -150 val_150 val_150 -152 val_152 val_152 -153 val_153 val_153 -155 val_155 val_155 -156 val_156 val_156 -157 val_157 val_157 -158 val_158 val_158 -160 val_160 val_160 -162 val_162 val_162 -163 val_163 val_163 -164 val_164 val_164 -165 val_165 val_165 -166 val_166 val_166 -167 val_167 val_167 -168 val_168 val_168 -169 val_169 val_169 -17 val_17 val_17 -170 val_170 val_170 -172 val_172 val_172 -174 val_174 val_174 -175 val_175 val_175 -176 val_176 val_176 -177 val_177 val_177 -178 val_178 val_178 -179 val_179 val_179 -18 val_18 val_18 -180 val_180 val_180 -181 val_181 val_181 -183 val_183 val_183 -186 val_186 val_186 -187 val_187 val_187 -189 val_189 val_189 -19 val_19 val_19 -190 val_190 val_190 -191 val_191 val_191 -192 val_192 val_192 -193 val_193 val_193 -194 val_194 val_194 -195 val_195 val_195 -196 val_196 val_196 -197 val_197 val_197 -199 val_199 val_199 -2 val_2 val_2 -20 val_20 val_20 -200 val_200 val_200 -201 val_201 val_201 -202 val_202 val_202 -203 val_203 val_203 -205 val_205 val_205 -207 val_207 val_207 -208 val_208 val_208 -209 val_209 val_209 -213 val_213 val_213 -214 val_214 val_214 -216 val_216 val_216 -217 val_217 val_217 -218 val_218 val_218 -219 val_219 val_219 -221 val_221 val_221 -222 val_222 val_222 -223 val_223 val_223 -224 val_224 val_224 -226 val_226 val_226 -228 val_228 val_228 -229 val_229 val_229 -230 val_230 val_230 -233 val_233 val_233 -235 val_235 val_235 -237 val_237 val_237 -238 val_238 val_238 -239 val_239 val_239 -24 val_24 val_24 -241 val_241 val_241 -242 val_242 val_242 -244 val_244 val_244 -247 val_247 val_247 -248 val_248 val_248 -249 val_249 val_249 -252 val_252 val_252 -255 val_255 val_255 -256 val_256 val_256 -257 val_257 val_257 -258 val_258 val_258 -26 val_26 val_26 -260 val_260 val_260 -262 val_262 val_262 -263 val_263 val_263 -265 val_265 val_265 -266 val_266 val_266 -27 val_27 val_27 -272 val_272 val_272 -273 val_273 val_273 -274 val_274 val_274 -275 val_275 val_275 -277 val_277 val_277 -278 val_278 val_278 -28 val_28 val_28 -280 val_280 val_280 -281 val_281 val_281 -282 val_282 val_282 -283 val_283 val_283 -284 val_284 val_284 -285 val_285 val_285 -286 val_286 val_286 -287 val_287 val_287 -288 val_288 val_288 -289 val_289 val_289 -291 val_291 val_291 -292 val_292 val_292 -296 val_296 val_296 -298 val_298 val_298 -30 val_30 val_30 -302 val_302 val_302 -305 val_305 val_305 -306 val_306 val_306 -307 val_307 val_307 -308 val_308 val_308 -309 val_309 val_309 -310 val_310 val_310 -311 val_311 val_311 -315 val_315 val_315 -316 val_316 val_316 -317 val_317 val_317 -318 val_318 val_318 -321 val_321 val_321 -322 val_322 val_322 -323 val_323 val_323 -325 val_325 val_325 -327 val_327 val_327 -33 val_33 val_33 -331 val_331 val_331 -332 val_332 val_332 -333 val_333 val_333 -335 val_335 val_335 -336 val_336 val_336 -338 val_338 val_338 -339 val_339 val_339 -34 val_34 val_34 -341 val_341 val_341 -342 val_342 val_342 -344 val_344 val_344 -345 val_345 val_345 -348 val_348 val_348 -35 val_35 val_35 -351 val_351 val_351 -353 val_353 val_353 -356 val_356 val_356 -360 val_360 val_360 -362 val_362 val_362 -364 val_364 val_364 -365 val_365 val_365 -366 val_366 val_366 -367 val_367 val_367 -368 val_368 val_368 -369 val_369 val_369 -37 val_37 val_37 -373 val_373 val_373 -374 val_374 val_374 -375 val_375 val_375 -377 val_377 val_377 -378 val_378 val_378 -379 val_379 val_379 -382 val_382 val_382 -384 val_384 val_384 -386 val_386 val_386 -389 val_389 val_389 -392 val_392 val_392 -393 val_393 val_393 -394 val_394 val_394 -395 val_395 val_395 -396 val_396 val_396 -397 val_397 val_397 -399 val_399 val_399 -4 val_4 val_4 -400 val_400 val_400 -401 val_401 val_401 -402 val_402 val_402 -403 val_403 val_403 -404 val_404 val_404 -406 val_406 val_406 -407 val_407 val_407 -409 val_409 val_409 -41 val_41 val_41 -411 val_411 val_411 -413 val_413 val_413 -414 val_414 val_414 -417 val_417 val_417 -418 val_418 val_418 -419 val_419 val_419 -42 val_42 val_42 -421 val_421 val_421 -424 val_424 val_424 -427 val_427 val_427 -429 val_429 val_429 -43 val_43 val_43 -430 val_430 val_430 -431 val_431 val_431 -432 val_432 val_432 -435 val_435 val_435 -436 val_436 val_436 -437 val_437 val_437 -438 val_438 val_438 -439 val_439 val_439 -44 val_44 val_44 -443 val_443 val_443 -444 val_444 val_444 -446 val_446 val_446 -448 val_448 val_448 -449 val_449 val_449 -452 val_452 val_452 -453 val_453 val_453 -454 val_454 val_454 -455 val_455 val_455 -457 val_457 val_457 -458 val_458 val_458 -459 val_459 val_459 -460 val_460 val_460 -462 val_462 val_462 -463 val_463 val_463 -466 val_466 val_466 -467 val_467 val_467 -468 val_468 val_468 -469 val_469 val_469 -47 val_47 val_47 -470 val_470 val_470 -472 val_472 val_472 -475 val_475 val_475 -477 val_477 val_477 -478 val_478 val_478 -479 val_479 val_479 -480 val_480 val_480 -481 val_481 val_481 -482 val_482 val_482 -483 val_483 val_483 -484 val_484 val_484 -485 val_485 val_485 -487 val_487 val_487 -489 val_489 val_489 -490 val_490 val_490 -491 val_491 val_491 -492 val_492 val_492 -493 val_493 val_493 -494 val_494 val_494 -495 val_495 val_495 -496 val_496 val_496 -497 val_497 val_497 -498 val_498 val_498 -5 val_5 val_5 -51 val_51 val_51 -53 val_53 val_53 -54 val_54 val_54 -57 val_57 val_57 -58 val_58 val_58 -64 val_64 val_64 -65 val_65 val_65 -66 val_66 val_66 -67 val_67 val_67 -69 val_69 val_69 -70 val_70 val_70 -72 val_72 val_72 -74 val_74 val_74 -76 val_76 val_76 -77 val_77 val_77 -78 val_78 val_78 -8 val_8 val_8 -80 val_80 val_80 -82 val_82 val_82 -83 val_83 val_83 -84 val_84 val_84 -85 val_85 val_85 -86 val_86 val_86 -87 val_87 val_87 -9 val_9 val_9 -90 val_90 val_90 -92 val_92 val_92 -95 val_95 val_95 -96 val_96 val_96 -97 val_97 val_97 -98 val_98 val_98 -tst1 500 500 -PREHOOK: query: explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -PREHOOK: type: QUERY -POSTHOOK: query: explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -309 -PREHOOK: query: explain -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -PREHOOK: type: QUERY -POSTHOOK: query: explain -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 10 <- Union 7 (CONTAINS) - Map 9 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: s4 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 66 Data size: 35904 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 66 Data size: 35904 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Union 3 - Vertex: Union 3 - Union 7 - Vertex: Union 7 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 val_0 0 val_0 -2 val_2 2 val_2 -4 val_4 4 val_4 -5 val_5 5 val_5 -8 val_8 8 val_8 -9 val_9 9 val_9 -tst1 500 tst1 500 -PREHOOK: query: explain -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -PREHOOK: type: QUERY -POSTHOOK: query: explain -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 10 <- Union 2 (CONTAINS) - Map 11 <- Union 4 (CONTAINS) - Map 12 <- Union 6 (CONTAINS) - Map 13 <- Union 8 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 9 <- Union 8 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '1' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 42500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: reverse(key) (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: src_thrift - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: astring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src_thrift - Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: lstring[0] (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 85 Data size: 16320 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 85 Data size: 16320 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 6 - Vertex: Union 6 - Union 8 - Vertex: Union 8 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src_thrift -#### A masked pattern was here #### -POSTHOOK: query: SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src_thrift -#### A masked pattern was here #### -03 1 -071 1 -09 1 -1 1 -10 1 -100 1 -104 1 -111 1 -114 1 -116 1 -123 1 -126 1 -128 1 -131 1 -134 1 -14 1 -152 1 -155 1 -169 1 -17 1 -170 1 -172 1 -178 1 -179 1 -182 1 -189 1 -19 1 -192 1 -194 1 -195 1 -199 1 -20 1 -207 1 -21 1 -213 1 -218 1 -223 1 -229 1 -233 1 -235 1 -24 1 -256 1 -257 1 -26 1 -265 1 -266 1 -27 1 -278 1 -280 1 -284 1 -291 1 -294 1 -30 1 -301 1 -302 1 -305 1 -312 1 -314 1 -316 1 -321 1 -323 1 -33 1 -333 1 -338 1 -344 1 -353 1 -356 1 -361 1 -364 1 -368 1 -369 1 -37 1 -377 1 -378 1 -386 1 -392 1 -397 1 -40 1 -404 1 -407 1 -411 1 -418 1 -419 1 -422 1 -424 1 -430 1 -432 1 -436 1 -437 1 -448 1 -454 1 -457 1 -46 1 -468 1 -477 1 -479 1 -490 1 -492 1 -493 1 -495 1 -496 1 -497 1 -501 1 -533 1 -534 1 -552 1 -57 1 -582 1 -611 1 -62 1 -652 1 -67 1 -671 1 -691 1 -712 1 -723 1 -73 1 -732 1 -75 1 -751 1 -761 1 -794 1 -8 1 -80 1 -814 1 -822 1 -831 1 -85 1 -854 1 -863 1 -871 1 -874 1 -882 1 -9 1 -902 1 -904 1 -91 1 -911 1 -92 1 -921 1 -922 1 -964 1 -97 1 -974 1 -982 1 -983 1 -record_7 1 -record_8 1 -NULL 1 -002 1 -02 1 -032 1 -034 1 -07 1 -074 1 -091 1 -105 1 -11 1 -113 1 -124 1 -136 1 -137 1 -138 1 -143 1 -150 1 -158 1 -160 1 -162 1 -165 1 -166 1 -174 1 -177 1 -18 1 -180 1 -186 1 -196 1 -197 1 -204 1 -217 1 -219 1 -226 1 -230 1 -238 1 -242 1 -248 1 -252 1 -260 1 -261 1 -263 1 -274 1 -277 1 -293 1 -296 1 -304 1 -307 1 -308 1 -325 1 -327 1 -342 1 -348 1 -35 1 -351 1 -367 1 -381 1 -389 1 -393 1 -394 1 -4 1 -403 1 -409 1 -412 1 -414 1 -421 1 -427 1 -429 1 -435 1 -438 1 -439 1 -45 1 -455 1 -458 1 -460 1 -461 1 -462 1 -471 1 -475 1 -478 1 -480 1 -484 1 -487 1 -50 1 -503 1 -513 1 -521 1 -523 1 -53 1 -532 1 -551 1 -56 1 -563 1 -571 1 -573 1 -58 1 -59 1 -591 1 -603 1 -604 1 -622 1 -633 1 -634 1 -641 1 -644 1 -651 1 -66 1 -661 1 -69 1 -693 1 -694 1 -703 1 -71 1 -731 1 -734 1 -752 1 -754 1 -764 1 -77 1 -772 1 -78 1 -784 1 -793 1 -803 1 -813 1 -83 1 -833 1 -843 1 -844 1 -864 1 -89 1 -90 1 -924 1 -934 1 -95 1 -954 1 -961 1 -993 1 -record_0 1 -record_2 1 -record_4 1 -record_6 1 -record_9 1 -001 1 -004 1 -01 1 -013 1 -051 1 -061 1 -062 1 -063 1 -102 1 -103 1 -120 1 -129 1 -133 1 -142 1 -145 1 -146 1 -149 1 -15 1 -157 1 -181 1 -187 1 -191 1 -2 1 -200 1 -201 1 -202 1 -203 1 -208 1 -214 1 -216 1 -222 1 -224 1 -228 1 -237 1 -239 1 -241 1 -244 1 -254 1 -255 1 -258 1 -262 1 -264 1 -273 1 -275 1 -28 1 -282 1 -285 1 -286 1 -287 1 -288 1 -29 1 -292 1 -298 1 -310 1 -311 1 -332 1 -335 1 -336 1 -339 1 -354 1 -360 1 -362 1 -373 1 -384 1 -395 1 -396 1 -402 1 -417 1 -42 1 -43 1 -442 1 -444 1 -449 1 -453 1 -459 1 -47 1 -48 1 -481 1 -482 1 -483 1 -485 1 -489 1 -494 1 -502 1 -541 1 -562 1 -574 1 -593 1 -60 1 -621 1 -631 1 -65 1 -653 1 -663 1 -664 1 -681 1 -682 1 -683 1 -713 1 -72 1 -74 1 -742 1 -76 1 -763 1 -771 1 -781 1 -79 1 -791 1 -811 1 -812 1 -82 1 -834 1 -852 1 -86 1 -861 1 -87 1 -872 1 -894 1 -903 1 -912 1 -914 1 -932 1 -942 1 -944 1 -96 1 -963 1 -record_5 1 -0 1 -021 1 -064 1 -08 1 -081 1 -082 1 -084 1 -094 1 -118 1 -119 1 -12 1 -122 1 -125 1 -153 1 -156 1 -163 1 -164 1 -167 1 -168 1 -175 1 -176 1 -183 1 -184 1 -190 1 -193 1 -205 1 -209 1 -221 1 -234 1 -243 1 -247 1 -249 1 -251 1 -271 1 -272 1 -281 1 -283 1 -289 1 -306 1 -309 1 -315 1 -317 1 -318 1 -322 1 -331 1 -34 1 -341 1 -345 1 -365 1 -366 1 -372 1 -374 1 -375 1 -379 1 -38 1 -382 1 -391 1 -399 1 -400 1 -401 1 -406 1 -41 1 -413 1 -431 1 -44 1 -443 1 -446 1 -452 1 -463 1 -466 1 -467 1 -469 1 -470 1 -472 1 -473 1 -491 1 -498 1 -5 1 -51 1 -54 1 -543 1 -554 1 -561 1 -572 1 -584 1 -594 1 -612 1 -613 1 -64 1 -662 1 -68 1 -692 1 -70 1 -702 1 -704 1 -714 1 -724 1 -773 1 -774 1 -782 1 -802 1 -81 1 -821 1 -832 1 -84 1 -842 1 -851 1 -873 1 -892 1 -933 1 -941 1 -971 1 -973 1 -98 1 -981 1 -984 1 -991 1 -record_1 1 -record_3 1 -PREHOOK: query: create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dst_union22 -POSTHOOK: query: create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dst_union22 -PREHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dst_union22_delta -POSTHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dst_union22_delta -PREHOOK: query: insert overwrite table dst_union22 partition (ds='1') -select key, value, key , value from src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dst_union22@ds=1 -POSTHOOK: query: insert overwrite table dst_union22 partition (ds='1') -select key, value, key , value from src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dst_union22@ds=1 -POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') -select key, key, value, key, value, value from src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dst_union22_delta@ds=1 -POSTHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') -select key, key, value, key, value, value from src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dst_union22_delta@ds=1 -POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain extended -insert overwrite table dst_union22 partition (ds='2') -select * from -( -select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 -UNION DISTINCT -select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 -from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on -a.k1 = b.k1 and a.ds='1' -where a.k1 > 20 -) -subq -PREHOOK: type: QUERY -POSTHOOK: query: explain extended -insert overwrite table dst_union22 partition (ds='2') -select * from -( -select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 -UNION DISTINCT -select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 -from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on -a.k1 = b.k1 and a.ds='1' -where a.k1 > 20 -) -subq -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 2 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: dst_union22_delta - Statistics: Num rows: 500 Data size: 454856 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(k0) <= 50.0) (type: boolean) - Statistics: Num rows: 166 Data size: 151012 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: k1 (type: string), k2 (type: string), k3 (type: string), k4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 166 Data size: 151012 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - null sort order: aaaa - sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 16936 - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 17436 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22_delta - name: default.dst_union22_delta - Truncated Path -> Alias: - /dst_union22_delta/ds=1 [dst_union22_delta] - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 278292 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(k1) > 20.0) (type: boolean) - Statistics: Num rows: 166 Data size: 89056 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: k1 (type: string), k2 (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 89056 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 89056 Basic stats: COMPLETE Column stats: PARTIAL - tag: 0 - value expressions: _col1 (type: string), _col2 (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 11124 - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11624 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - name: default.dst_union22 - Truncated Path -> Alias: - /dst_union22/ds=1 [a] - Map 6 - Map Operator Tree: - TableScan - alias: dst_union22_delta - Statistics: Num rows: 500 Data size: 367272 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: ((UDFToDouble(k0) > 50.0) and (UDFToDouble(k1) > 20.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 40399 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: k1 (type: string), k3 (type: string), k4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 55 Data size: 40399 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 40399 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: _col1 (type: string), _col2 (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 16936 - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 17436 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22_delta - name: default.dst_union22_delta - Truncated Path -> Alias: - /dst_union22_delta/ds=1 [dst_union22_delta] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 174 Data size: 124486 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=2/ - Statistics: Num rows: 174 Data size: 124486 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join 0 to 1 - filter mappings: - 0 [1, 1] - filter predicates: - 0 {(VALUE._col1 = '1')} - 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 182 Data size: 97961 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 182 Data size: 97961 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - null sort order: aaaa - sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Union 2 - Vertex: Union 2 - - Stage: Stage-2 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - partition: - ds 2 - replace: true -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - - Stage: Stage-3 - Stats Work - Basic Stats Work: -#### A masked pattern was here #### - -PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') -select * from -( -select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 -UNION DISTINCT -select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 -from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on -a.k1 = b.k1 and a.ds='1' -where a.k1 > 20 -) -subq -PREHOOK: type: QUERY -PREHOOK: Input: default@dst_union22 -PREHOOK: Input: default@dst_union22@ds=1 -PREHOOK: Input: default@dst_union22_delta -PREHOOK: Input: default@dst_union22_delta@ds=1 -PREHOOK: Output: default@dst_union22@ds=2 -POSTHOOK: query: insert overwrite table dst_union22 partition (ds='2') -select * from -( -select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 -UNION DISTINCT -select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 -from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on -a.k1 = b.k1 and a.ds='1' -where a.k1 > 20 -) -subq -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dst_union22 -POSTHOOK: Input: default@dst_union22@ds=1 -POSTHOOK: Input: default@dst_union22_delta -POSTHOOK: Input: default@dst_union22_delta@ds=1 -POSTHOOK: Output: default@dst_union22@ds=2 -POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k1 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k1, type:string, comment:null), (dst_union22)a.FieldSchema(name:k1, type:string, comment:null), ] -POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k2 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k2, type:string, comment:null), (dst_union22)a.FieldSchema(name:k2, type:string, comment:null), ] -POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k3 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), ] -POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k4 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), ] -PREHOOK: query: select * from dst_union22 where ds = '2' -PREHOOK: type: QUERY -PREHOOK: Input: default@dst_union22 -PREHOOK: Input: default@dst_union22@ds=2 -#### A masked pattern was here #### -POSTHOOK: query: select * from dst_union22 where ds = '2' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dst_union22 -POSTHOOK: Input: default@dst_union22@ds=2 -#### A masked pattern was here #### -103 val_103 103 val_103 2 -113 val_113 113 val_113 2 -120 val_120 120 val_120 2 -128 val_128 128 val_128 2 -137 val_137 137 val_137 2 -145 val_145 145 val_145 2 -146 val_146 146 val_146 2 -15 val_15 15 val_15 2 -156 val_156 156 val_156 2 -163 val_163 163 val_163 2 -165 val_165 165 val_165 2 -166 val_166 166 val_166 2 -167 val_167 167 val_167 2 -170 val_170 170 val_170 2 -174 val_174 174 val_174 2 -178 val_178 178 val_178 2 -183 val_183 183 val_183 2 -191 val_191 191 val_191 2 -195 val_195 195 val_195 2 -199 val_199 199 val_199 2 -2 val_2 2 val_2 2 -203 val_203 203 val_203 2 -217 val_217 217 val_217 2 -224 val_224 224 val_224 2 -229 val_229 229 val_229 2 -242 val_242 242 val_242 2 -27 val_27 NULL NULL 2 -272 val_272 272 val_272 2 -28 val_28 NULL NULL 2 -280 val_280 280 val_280 2 -282 val_282 282 val_282 2 -283 val_283 283 val_283 2 -287 val_287 287 val_287 2 -289 val_289 289 val_289 2 -292 val_292 292 val_292 2 -306 val_306 306 val_306 2 -307 val_307 307 val_307 2 -309 val_309 309 val_309 2 -310 val_310 310 val_310 2 -315 val_315 315 val_315 2 -327 val_327 327 val_327 2 -33 val_33 33 val_33 2 -331 val_331 331 val_331 2 -332 val_332 332 val_332 2 -333 val_333 333 val_333 2 -338 val_338 338 val_338 2 -344 val_344 344 val_344 2 -348 val_348 348 val_348 2 -35 val_35 35 val_35 2 -353 val_353 353 val_353 2 -362 val_362 362 val_362 2 -37 val_37 37 val_37 2 -373 val_373 373 val_373 2 -379 val_379 379 val_379 2 -4 val_4 4 val_4 2 -400 val_400 400 val_400 2 -401 val_401 401 val_401 2 -403 val_403 403 val_403 2 -406 val_406 406 val_406 2 -41 val_41 41 val_41 2 -414 val_414 414 val_414 2 -417 val_417 417 val_417 2 -431 val_431 431 val_431 2 -432 val_432 432 val_432 2 -436 val_436 436 val_436 2 -439 val_439 439 val_439 2 -452 val_452 452 val_452 2 -453 val_453 453 val_453 2 -455 val_455 455 val_455 2 -466 val_466 466 val_466 2 -469 val_469 469 val_469 2 -47 val_47 NULL NULL 2 -470 val_470 470 val_470 2 -475 val_475 475 val_475 2 -477 val_477 477 val_477 2 -493 val_493 493 val_493 2 -496 val_496 496 val_496 2 -54 val_54 54 val_54 2 -57 val_57 57 val_57 2 -65 val_65 65 val_65 2 -69 val_69 69 val_69 2 -77 val_77 77 val_77 2 -78 val_78 78 val_78 2 -8 val_8 8 val_8 2 -82 val_82 82 val_82 2 -87 val_87 87 val_87 2 -9 val_9 9 val_9 2 -92 val_92 92 val_92 2 -10 val_10 10 val_10 2 -11 val_11 11 val_11 2 -111 val_111 111 val_111 2 -114 val_114 114 val_114 2 -116 val_116 116 val_116 2 -12 val_12 12 val_12 2 -129 val_129 129 val_129 2 -136 val_136 136 val_136 2 -152 val_152 152 val_152 2 -155 val_155 155 val_155 2 -158 val_158 158 val_158 2 -168 val_168 168 val_168 2 -176 val_176 176 val_176 2 -177 val_177 177 val_177 2 -179 val_179 179 val_179 2 -18 val_18 18 val_18 2 -186 val_186 186 val_186 2 -194 val_194 194 val_194 2 -208 val_208 208 val_208 2 -214 val_214 214 val_214 2 -222 val_222 222 val_222 2 -226 val_226 226 val_226 2 -230 val_230 230 val_230 2 -237 val_237 237 val_237 2 -24 val_24 NULL NULL 2 -24 val_24 24 val_24 2 -248 val_248 248 val_248 2 -255 val_255 255 val_255 2 -256 val_256 256 val_256 2 -26 val_26 26 val_26 2 -265 val_265 265 val_265 2 -266 val_266 266 val_266 2 -273 val_273 273 val_273 2 -275 val_275 275 val_275 2 -28 val_28 28 val_28 2 -284 val_284 284 val_284 2 -291 val_291 291 val_291 2 -298 val_298 298 val_298 2 -30 val_30 NULL NULL 2 -30 val_30 30 val_30 2 -308 val_308 308 val_308 2 -321 val_321 321 val_321 2 -322 val_322 322 val_322 2 -335 val_335 335 val_335 2 -336 val_336 336 val_336 2 -34 val_34 34 val_34 2 -341 val_341 341 val_341 2 -342 val_342 342 val_342 2 -369 val_369 369 val_369 2 -374 val_374 374 val_374 2 -377 val_377 377 val_377 2 -382 val_382 382 val_382 2 -386 val_386 386 val_386 2 -393 val_393 393 val_393 2 -397 val_397 397 val_397 2 -402 val_402 402 val_402 2 -404 val_404 404 val_404 2 -407 val_407 407 val_407 2 -418 val_418 418 val_418 2 -446 val_446 446 val_446 2 -448 val_448 448 val_448 2 -457 val_457 457 val_457 2 -459 val_459 459 val_459 2 -462 val_462 462 val_462 2 -463 val_463 463 val_463 2 -472 val_472 472 val_472 2 -478 val_478 478 val_478 2 -479 val_479 479 val_479 2 -480 val_480 480 val_480 2 -483 val_483 483 val_483 2 -485 val_485 485 val_485 2 -491 val_491 491 val_491 2 -5 val_5 5 val_5 2 -53 val_53 53 val_53 2 -67 val_67 67 val_67 2 -76 val_76 76 val_76 2 -83 val_83 83 val_83 2 -86 val_86 86 val_86 2 -95 val_95 95 val_95 2 -96 val_96 96 val_96 2 -98 val_98 98 val_98 2 -100 val_100 100 val_100 2 -125 val_125 125 val_125 2 -126 val_126 126 val_126 2 -134 val_134 134 val_134 2 -138 val_138 138 val_138 2 -143 val_143 143 val_143 2 -149 val_149 149 val_149 2 -150 val_150 150 val_150 2 -157 val_157 157 val_157 2 -164 val_164 164 val_164 2 -169 val_169 169 val_169 2 -17 val_17 17 val_17 2 -172 val_172 172 val_172 2 -181 val_181 181 val_181 2 -187 val_187 187 val_187 2 -189 val_189 189 val_189 2 -190 val_190 190 val_190 2 -192 val_192 192 val_192 2 -197 val_197 197 val_197 2 -201 val_201 201 val_201 2 -202 val_202 202 val_202 2 -205 val_205 205 val_205 2 -207 val_207 207 val_207 2 -209 val_209 209 val_209 2 -216 val_216 216 val_216 2 -218 val_218 218 val_218 2 -223 val_223 223 val_223 2 -228 val_228 228 val_228 2 -233 val_233 233 val_233 2 -235 val_235 235 val_235 2 -238 val_238 238 val_238 2 -241 val_241 241 val_241 2 -244 val_244 244 val_244 2 -247 val_247 247 val_247 2 -252 val_252 252 val_252 2 -258 val_258 258 val_258 2 -260 val_260 260 val_260 2 -262 val_262 262 val_262 2 -27 val_27 27 val_27 2 -274 val_274 274 val_274 2 -277 val_277 277 val_277 2 -278 val_278 278 val_278 2 -288 val_288 288 val_288 2 -296 val_296 296 val_296 2 -302 val_302 302 val_302 2 -305 val_305 305 val_305 2 -311 val_311 311 val_311 2 -318 val_318 318 val_318 2 -345 val_345 345 val_345 2 -35 val_35 NULL NULL 2 -356 val_356 356 val_356 2 -360 val_360 360 val_360 2 -365 val_365 365 val_365 2 -375 val_375 375 val_375 2 -389 val_389 389 val_389 2 -392 val_392 392 val_392 2 -394 val_394 394 val_394 2 -395 val_395 395 val_395 2 -409 val_409 409 val_409 2 -41 val_41 NULL NULL 2 -411 val_411 411 val_411 2 -413 val_413 413 val_413 2 -419 val_419 419 val_419 2 -42 val_42 42 val_42 2 -427 val_427 427 val_427 2 -429 val_429 429 val_429 2 -430 val_430 430 val_430 2 -44 val_44 NULL NULL 2 -444 val_444 444 val_444 2 -454 val_454 454 val_454 2 -460 val_460 460 val_460 2 -468 val_468 468 val_468 2 -47 val_47 47 val_47 2 -481 val_481 481 val_481 2 -482 val_482 482 val_482 2 -484 val_484 484 val_484 2 -489 val_489 489 val_489 2 -492 val_492 492 val_492 2 -495 val_495 495 val_495 2 -498 val_498 498 val_498 2 -58 val_58 58 val_58 2 -64 val_64 64 val_64 2 -66 val_66 66 val_66 2 -70 val_70 70 val_70 2 -72 val_72 72 val_72 2 -74 val_74 74 val_74 2 -80 val_80 80 val_80 2 -84 val_84 84 val_84 2 -90 val_90 90 val_90 2 -0 val_0 0 val_0 2 -104 val_104 104 val_104 2 -105 val_105 105 val_105 2 -118 val_118 118 val_118 2 -119 val_119 119 val_119 2 -131 val_131 131 val_131 2 -133 val_133 133 val_133 2 -153 val_153 153 val_153 2 -160 val_160 160 val_160 2 -162 val_162 162 val_162 2 -175 val_175 175 val_175 2 -180 val_180 180 val_180 2 -19 val_19 19 val_19 2 -193 val_193 193 val_193 2 -196 val_196 196 val_196 2 -20 val_20 20 val_20 2 -200 val_200 200 val_200 2 -213 val_213 213 val_213 2 -219 val_219 219 val_219 2 -221 val_221 221 val_221 2 -239 val_239 239 val_239 2 -249 val_249 249 val_249 2 -257 val_257 257 val_257 2 -26 val_26 NULL NULL 2 -263 val_263 263 val_263 2 -281 val_281 281 val_281 2 -285 val_285 285 val_285 2 -286 val_286 286 val_286 2 -316 val_316 316 val_316 2 -317 val_317 317 val_317 2 -323 val_323 323 val_323 2 -325 val_325 325 val_325 2 -33 val_33 NULL NULL 2 -339 val_339 339 val_339 2 -34 val_34 NULL NULL 2 -351 val_351 351 val_351 2 -364 val_364 364 val_364 2 -366 val_366 366 val_366 2 -367 val_367 367 val_367 2 -368 val_368 368 val_368 2 -37 val_37 NULL NULL 2 -378 val_378 378 val_378 2 -384 val_384 384 val_384 2 -396 val_396 396 val_396 2 -399 val_399 399 val_399 2 -42 val_42 NULL NULL 2 -421 val_421 421 val_421 2 -424 val_424 424 val_424 2 -43 val_43 NULL NULL 2 -43 val_43 43 val_43 2 -435 val_435 435 val_435 2 -437 val_437 437 val_437 2 -438 val_438 438 val_438 2 -44 val_44 44 val_44 2 -443 val_443 443 val_443 2 -449 val_449 449 val_449 2 -458 val_458 458 val_458 2 -467 val_467 467 val_467 2 -487 val_487 487 val_487 2 -490 val_490 490 val_490 2 -494 val_494 494 val_494 2 -497 val_497 497 val_497 2 -51 val_51 51 val_51 2 -85 val_85 85 val_85 2 -97 val_97 97 val_97 2 -PREHOOK: query: explain -select s.key2, s.value2 -from ( - select transform(key, value) using 'cat' as (key2, value2) - from src - UNION DISTINCT - select key as key2, value as value2 from src) s -PREHOOK: type: QUERY -POSTHOOK: query: explain -select s.key2, s.value2 -from ( - select transform(key, value) using 'cat' as (key2, value2) - from src - UNION DISTINCT - select key as key2, value as value2 from src) s -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select s.key2, s.value2 -from ( - select transform(key, value) using 'cat' as (key2, value2) - from src - UNION DISTINCT - select key as key2, value as value2 from src) s -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select s.key2, s.value2 -from ( - select transform(key, value) using 'cat' as (key2, value2) - from src - UNION DISTINCT - select key as key2, value as value2 from src) s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 val_0 -103 val_103 -105 val_105 -116 val_116 -120 val_120 -129 val_129 -131 val_131 -136 val_136 -155 val_155 -157 val_157 -160 val_160 -180 val_180 -183 val_183 -190 val_190 -191 val_191 -193 val_193 -205 val_205 -207 val_207 -213 val_213 -223 val_223 -235 val_235 -238 val_238 -239 val_239 -24 val_24 -248 val_248 -260 val_260 -28 val_28 -286 val_286 -298 val_298 -305 val_305 -306 val_306 -307 val_307 -308 val_308 -315 val_315 -317 val_317 -321 val_321 -33 val_33 -331 val_331 -356 val_356 -360 val_360 -366 val_366 -367 val_367 -368 val_368 -37 val_37 -378 val_378 -379 val_379 -382 val_382 -389 val_389 -395 val_395 -400 val_400 -402 val_402 -406 val_406 -407 val_407 -42 val_42 -430 val_430 -436 val_436 -44 val_44 -452 val_452 -466 val_466 -467 val_467 -472 val_472 -480 val_480 -484 val_484 -485 val_485 -487 val_487 -495 val_495 -496 val_496 -498 val_498 -51 val_51 -57 val_57 -58 val_58 -65 val_65 -66 val_66 -69 val_69 -70 val_70 -78 val_78 -8 val_8 -92 val_92 -96 val_96 -104 val_104 -111 val_111 -114 val_114 -118 val_118 -12 val_12 -133 val_133 -138 val_138 -153 val_153 -156 val_156 -162 val_162 -168 val_168 -174 val_174 -176 val_176 -178 val_178 -181 val_181 -186 val_186 -187 val_187 -192 val_192 -197 val_197 -199 val_199 -2 val_2 -201 val_201 -203 val_203 -208 val_208 -214 val_214 -218 val_218 -219 val_219 -230 val_230 -237 val_237 -249 val_249 -256 val_256 -257 val_257 -277 val_277 -278 val_278 -284 val_284 -285 val_285 -310 val_310 -322 val_322 -327 val_327 -333 val_333 -336 val_336 -338 val_338 -341 val_341 -344 val_344 -348 val_348 -351 val_351 -362 val_362 -364 val_364 -374 val_374 -392 val_392 -397 val_397 -4 val_4 -403 val_403 -404 val_404 -419 val_419 -421 val_421 -427 val_427 -429 val_429 -437 val_437 -438 val_438 -439 val_439 -454 val_454 -455 val_455 -460 val_460 -463 val_463 -470 val_470 -478 val_478 -479 val_479 -489 val_489 -491 val_491 -492 val_492 -53 val_53 -67 val_67 -82 val_82 -90 val_90 -97 val_97 -10 val_10 -125 val_125 -134 val_134 -143 val_143 -145 val_145 -149 val_149 -150 val_150 -158 val_158 -163 val_163 -164 val_164 -166 val_166 -17 val_17 -170 val_170 -172 val_172 -189 val_189 -19 val_19 -195 val_195 -196 val_196 -20 val_20 -209 val_209 -216 val_216 -217 val_217 -224 val_224 -228 val_228 -229 val_229 -233 val_233 -241 val_241 -244 val_244 -247 val_247 -255 val_255 -258 val_258 -26 val_26 -263 val_263 -265 val_265 -266 val_266 -272 val_272 -273 val_273 -274 val_274 -281 val_281 -291 val_291 -296 val_296 -30 val_30 -302 val_302 -309 val_309 -316 val_316 -318 val_318 -325 val_325 -332 val_332 -335 val_335 -339 val_339 -342 val_342 -345 val_345 -353 val_353 -369 val_369 -373 val_373 -375 val_375 -377 val_377 -384 val_384 -386 val_386 -394 val_394 -396 val_396 -399 val_399 -401 val_401 -41 val_41 -413 val_413 -414 val_414 -431 val_431 -446 val_446 -448 val_448 -449 val_449 -459 val_459 -462 val_462 -468 val_468 -47 val_47 -482 val_482 -490 val_490 -493 val_493 -494 val_494 -497 val_497 -5 val_5 -54 val_54 -74 val_74 -77 val_77 -80 val_80 -84 val_84 -85 val_85 -87 val_87 -9 val_9 -95 val_95 -100 val_100 -11 val_11 -113 val_113 -119 val_119 -126 val_126 -128 val_128 -137 val_137 -146 val_146 -15 val_15 -152 val_152 -165 val_165 -167 val_167 -169 val_169 -175 val_175 -177 val_177 -179 val_179 -18 val_18 -194 val_194 -200 val_200 -202 val_202 -221 val_221 -222 val_222 -226 val_226 -242 val_242 -252 val_252 -262 val_262 -27 val_27 -275 val_275 -280 val_280 -282 val_282 -283 val_283 -287 val_287 -288 val_288 -289 val_289 -292 val_292 -311 val_311 -323 val_323 -34 val_34 -35 val_35 -365 val_365 -393 val_393 -409 val_409 -411 val_411 -417 val_417 -418 val_418 -424 val_424 -43 val_43 -432 val_432 -435 val_435 -443 val_443 -444 val_444 -453 val_453 -457 val_457 -458 val_458 -469 val_469 -475 val_475 -477 val_477 -481 val_481 -483 val_483 -64 val_64 -72 val_72 -76 val_76 -83 val_83 -86 val_86 -98 val_98 -PREHOOK: query: create table src2 as select key, count(1) as count from src group by key -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@src2 -POSTHOOK: query: create table src2 as select key, count(1) as count from src group by key -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src2 -POSTHOOK: Lineage: src2.count EXPRESSION [(src)src.null, ] -POSTHOOK: Lineage: src2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: create table src3 as select * from src2 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src2 -PREHOOK: Output: database:default -PREHOOK: Output: default@src3 -POSTHOOK: query: create table src3 as select * from src2 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src3 -POSTHOOK: Lineage: src3.count SIMPLE [(src2)src2.FieldSchema(name:count, type:bigint, comment:null), ] -POSTHOOK: Lineage: src3.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ] -PREHOOK: query: create table src4 as select * from src2 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src2 -PREHOOK: Output: database:default -PREHOOK: Output: default@src4 -POSTHOOK: query: create table src4 as select * from src2 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src4 -POSTHOOK: Lineage: src4.count SIMPLE [(src2)src2.FieldSchema(name:count, type:bigint, comment:null), ] -POSTHOOK: Lineage: src4.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ] -PREHOOK: query: create table src5 as select * from src2 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src2 -PREHOOK: Output: database:default -PREHOOK: Output: default@src5 -POSTHOOK: query: create table src5 as select * from src2 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src5 -POSTHOOK: Lineage: src5.count SIMPLE [(src2)src2.FieldSchema(name:count, type:bigint, comment:null), ] -POSTHOOK: Lineage: src5.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ] -PREHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select key, count from src4 where key < 10 - UNION DISTINCT - select key, count(1) as count from src5 where key < 10 group by key -)s -PREHOOK: type: QUERY -POSTHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select key, count from src4 where key < 10 - UNION DISTINCT - select key, count(1) as count from src5 where key < 10 group by key -)s -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 8 <- Union 2 (CONTAINS) - Map 9 <- Union 4 (CONTAINS) - Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src2 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2 - numFiles 4 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2 - numFiles 4 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src2 - name: default.src2 - Truncated Path -> Alias: - /src2 [src2] - Map 10 - Map Operator Tree: - TableScan - alias: src5 - Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: bigint) - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src5 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src5 - name: default.src5 - Truncated Path -> Alias: - /src5 [src5] - Map 8 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src3 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src3 - name: default.src3 - Truncated Path -> Alias: - /src3 [src3] - Map 9 - Map Operator Tree: - TableScan - alias: src4 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src4 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src4 - name: default.src4 - Truncated Path -> Alias: - /src4 [src4] - Reducer 11 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 51 Data size: 8958 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 7 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 77 Data size: 13919 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 77 Data size: 13919 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 6 - Vertex: Union 6 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select key, count from src4 where key < 10 - UNION DISTINCT - select key, count(1) as count from src5 where key < 10 group by key -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2 -PREHOOK: Input: default@src3 -PREHOOK: Input: default@src4 -PREHOOK: Input: default@src5 -#### A masked pattern was here #### -POSTHOOK: query: select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select key, count from src4 where key < 10 - UNION DISTINCT - select key, count(1) as count from src5 where key < 10 group by key -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2 -POSTHOOK: Input: default@src3 -POSTHOOK: Input: default@src4 -POSTHOOK: Input: default@src5 -#### A masked pattern was here #### -9 1 -0 3 -4 1 -2 1 -0 1 -5 1 -5 3 -8 1 -PREHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 -)s -PREHOOK: type: QUERY -POSTHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 -)s -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE), Union 4 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src2 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2 - numFiles 4 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2 - numFiles 4 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src2 - name: default.src2 - Truncated Path -> Alias: - /src2 [src2] - Map 6 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src3 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src3 - name: default.src3 - Truncated Path -> Alias: - /src3 [src3] - Map 7 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src4 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src4 - name: default.src4 - Truncated Path -> Alias: - /src4 [a] - Map 9 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: _col1 (type: bigint) - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src5 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src5 - name: default.src5 - Truncated Path -> Alias: - /src5 [b] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 108 Data size: 19391 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 108 Data size: 19391 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 8 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col2 - Position of Big Table: 1 - Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2 -PREHOOK: Input: default@src3 -PREHOOK: Input: default@src4 -PREHOOK: Input: default@src5 -#### A masked pattern was here #### -POSTHOOK: query: select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2 -POSTHOOK: Input: default@src3 -POSTHOOK: Input: default@src4 -POSTHOOK: Input: default@src5 -#### A masked pattern was here #### -9 1 -0 3 -4 1 -2 1 -5 3 -8 1 -PREHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key -)s -PREHOOK: type: QUERY -POSTHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key -)s -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 4 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src2 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2 - numFiles 4 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2 - numFiles 4 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src2 - name: default.src2 - Truncated Path -> Alias: - /src2 [src2] - Map 10 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src5 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src5 - name: default.src5 - Truncated Path -> Alias: - /src5 [b] - Map 6 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src3 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src3 - name: default.src3 - Truncated Path -> Alias: - /src3 [src3] - Map 7 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src4 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src4 - name: default.src4 - Truncated Path -> Alias: - /src4 [a] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 79 Data size: 14280 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 79 Data size: 14280 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 8 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Position of Big Table: 0 - Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: bigint) - auto parallelism: true - Reducer 9 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 9862 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2 -PREHOOK: Input: default@src3 -PREHOOK: Input: default@src4 -PREHOOK: Input: default@src5 -#### A masked pattern was here #### -POSTHOOK: query: select s.key, s.count from ( - select key, count from src2 where key < 10 - UNION DISTINCT - select key, count from src3 where key < 10 - UNION DISTINCT - select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2 -POSTHOOK: Input: default@src3 -POSTHOOK: Input: default@src4 -POSTHOOK: Input: default@src5 -#### A masked pattern was here #### -9 1 -0 3 -4 1 -2 1 -0 1 -5 1 -5 3 -8 1 -PREHOOK: query: create table tmp_srcpart like srcpart -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tmp_srcpart -POSTHOOK: query: create table tmp_srcpart like srcpart -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tmp_srcpart -PREHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') -select key, value from srcpart where ds='2008-04-08' and hr='11' -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 -POSTHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') -select key, value from srcpart where ds='2008-04-08' and hr='11' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain -create table tmp_unionall as -SELECT count(1) as counts, key, value -FROM -( - SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - - UNION DISTINCT - - SELECT key, key as value FROM ( - SELECT distinct key FROM ( - SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - UNION DISTINCT - SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' - )t - ) master_table -) a GROUP BY key, value -PREHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: query: explain -create table tmp_unionall as -SELECT count(1) as counts, key, value -FROM -( - SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - - UNION DISTINCT - - SELECT key, key as value FROM ( - SELECT distinct key FROM ( - SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - UNION DISTINCT - SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' - )t - ) master_table -) a GROUP BY key, value -POSTHOOK: type: CREATETABLE_AS_SELECT -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-4 depends on stages: Stage-2, Stage-0 - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count() - keys: _col0 (type: string), _col1 (type: string) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 375 Data size: 69750 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 375 Data size: 69750 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 375 Data size: 69750 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmp_unionall - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 90240 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 90240 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 - - Stage: Stage-2 - Dependency Collection - - Stage: Stage-4 - Create Table Operator: - Create Table - columns: counts bigint, key string, value string - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmp_unionall - - Stage: Stage-3 - Stats Work - Basic Stats Work: - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: EXPLAIN -SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - input vertices: - 1 Map 4 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Lateral View Forward - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - UDTF Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - function name: explode - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 775 Data size: 8233 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -1 0 val_0 -1 103 val_103 -1 105 val_105 -1 116 val_116 -1 120 val_120 -1 129 val_129 -1 131 val_131 -1 136 val_136 -1 155 val_155 -1 157 val_157 -1 160 val_160 -1 180 val_180 -1 183 val_183 -1 190 val_190 -1 191 val_191 -1 193 val_193 -1 205 val_205 -1 207 val_207 -1 213 val_213 -1 223 val_223 -1 235 val_235 -1 238 val_238 -1 239 val_239 -1 24 val_24 -1 248 val_248 -1 260 val_260 -1 28 val_28 -1 286 val_286 -1 298 val_298 -1 305 val_305 -1 306 val_306 -1 307 val_307 -1 308 val_308 -1 315 val_315 -1 317 val_317 -1 321 val_321 -1 33 val_33 -1 331 val_331 -1 356 val_356 -1 360 val_360 -1 366 val_366 -1 367 val_367 -1 368 val_368 -1 37 val_37 -1 378 val_378 -1 379 val_379 -1 382 val_382 -1 389 val_389 -1 395 val_395 -1 400 val_400 -1 402 val_402 -1 406 val_406 -1 407 val_407 -1 42 val_42 -1 430 val_430 -1 436 val_436 -1 44 val_44 -1 452 val_452 -1 466 val_466 -1 467 val_467 -1 472 val_472 -1 480 val_480 -1 484 val_484 -1 485 val_485 -1 487 val_487 -1 495 val_495 -1 496 val_496 -1 498 val_498 -1 51 val_51 -1 57 val_57 -1 58 val_58 -1 65 val_65 -1 66 val_66 -1 69 val_69 -1 70 val_70 -1 78 val_78 -1 8 val_8 -1 92 val_92 -1 96 val_96 -1 104 val_104 -1 111 val_111 -1 114 val_114 -1 118 val_118 -1 12 val_12 -1 133 val_133 -1 138 val_138 -1 153 val_153 -1 156 val_156 -1 162 val_162 -1 168 val_168 -1 174 val_174 -1 176 val_176 -1 178 val_178 -1 181 val_181 -1 186 val_186 -1 187 val_187 -1 192 val_192 -1 197 val_197 -1 199 val_199 -1 2 val_2 -1 201 val_201 -1 203 val_203 -1 208 val_208 -1 214 val_214 -1 218 val_218 -1 219 val_219 -1 230 val_230 -1 237 val_237 -1 249 val_249 -1 256 val_256 -1 257 val_257 -1 277 val_277 -1 278 val_278 -1 284 val_284 -1 285 val_285 -1 310 val_310 -1 322 val_322 -1 327 val_327 -1 333 val_333 -1 336 val_336 -1 338 val_338 -1 341 val_341 -1 344 val_344 -1 348 val_348 -1 351 val_351 -1 362 val_362 -1 364 val_364 -1 374 val_374 -1 392 val_392 -1 397 val_397 -1 4 val_4 -1 403 val_403 -1 404 val_404 -1 419 val_419 -1 421 val_421 -1 427 val_427 -1 429 val_429 -1 437 val_437 -1 438 val_438 -1 439 val_439 -1 454 val_454 -1 455 val_455 -1 460 val_460 -1 463 val_463 -1 470 val_470 -1 478 val_478 -1 479 val_479 -1 489 val_489 -1 491 val_491 -1 492 val_492 -1 53 val_53 -1 67 val_67 -1 82 val_82 -1 90 val_90 -1 97 val_97 -1 10 val_10 -1 125 val_125 -1 134 val_134 -1 143 val_143 -1 145 val_145 -1 149 val_149 -1 150 val_150 -1 158 val_158 -1 163 val_163 -1 164 val_164 -1 166 val_166 -1 17 val_17 -1 170 val_170 -1 172 val_172 -1 189 val_189 -1 19 val_19 -1 195 val_195 -1 196 val_196 -1 20 val_20 -1 209 val_209 -1 216 val_216 -1 217 val_217 -1 224 val_224 -1 228 val_228 -1 229 val_229 -1 233 val_233 -1 241 val_241 -1 244 val_244 -1 247 val_247 -1 255 val_255 -1 258 val_258 -1 26 val_26 -1 263 val_263 -1 265 val_265 -1 266 val_266 -1 272 val_272 -1 273 val_273 -1 274 val_274 -1 281 val_281 -1 291 val_291 -1 296 val_296 -1 30 val_30 -1 302 val_302 -1 309 val_309 -1 316 val_316 -1 318 val_318 -1 325 val_325 -1 332 val_332 -1 335 val_335 -1 339 val_339 -1 342 val_342 -1 345 val_345 -1 353 val_353 -1 369 val_369 -1 373 val_373 -1 375 val_375 -1 377 val_377 -1 384 val_384 -1 386 val_386 -1 394 val_394 -1 396 val_396 -1 399 val_399 -1 401 val_401 -1 41 val_41 -1 413 val_413 -1 414 val_414 -1 431 val_431 -1 446 val_446 -1 448 val_448 -1 449 val_449 -1 459 val_459 -1 462 val_462 -1 468 val_468 -1 47 val_47 -1 482 val_482 -1 490 val_490 -1 493 val_493 -1 494 val_494 -1 497 val_497 -1 5 val_5 -1 54 val_54 -1 74 val_74 -1 77 val_77 -1 80 val_80 -1 84 val_84 -1 85 val_85 -1 87 val_87 -1 9 val_9 -1 95 val_95 -1 100 val_100 -1 11 val_11 -1 113 val_113 -1 119 val_119 -1 126 val_126 -1 128 val_128 -1 137 val_137 -1 146 val_146 -1 15 val_15 -1 152 val_152 -1 165 val_165 -1 167 val_167 -1 169 val_169 -1 175 val_175 -1 177 val_177 -1 179 val_179 -1 18 val_18 -1 194 val_194 -1 200 val_200 -1 202 val_202 -1 221 val_221 -1 222 val_222 -1 226 val_226 -1 242 val_242 -1 252 val_252 -1 262 val_262 -1 27 val_27 -1 275 val_275 -1 280 val_280 -1 282 val_282 -1 283 val_283 -1 287 val_287 -1 288 val_288 -1 289 val_289 -1 292 val_292 -1 311 val_311 -1 323 val_323 -1 34 val_34 -1 35 val_35 -1 365 val_365 -1 393 val_393 -1 409 val_409 -1 411 val_411 -1 417 val_417 -1 418 val_418 -1 424 val_424 -1 43 val_43 -1 432 val_432 -1 435 val_435 -1 443 val_443 -1 444 val_444 -1 453 val_453 -1 457 val_457 -1 458 val_458 -1 469 val_469 -1 475 val_475 -1 477 val_477 -1 481 val_481 -1 483 val_483 -1 64 val_64 -1 72 val_72 -1 76 val_76 -1 83 val_83 -1 86 val_86 -1 98 val_98 -PREHOOK: query: SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -1 0 val_0 -1 103 val_103 -1 105 val_105 -1 116 val_116 -1 120 val_120 -1 129 val_129 -1 131 val_131 -1 136 val_136 -1 155 val_155 -1 157 val_157 -1 160 val_160 -1 180 val_180 -1 183 val_183 -1 190 val_190 -1 191 val_191 -1 193 val_193 -1 205 val_205 -1 207 val_207 -1 213 val_213 -1 223 val_223 -1 235 val_235 -1 238 val_238 -1 239 val_239 -1 24 val_24 -1 248 val_248 -1 260 val_260 -1 28 val_28 -1 286 val_286 -1 298 val_298 -1 305 val_305 -1 306 val_306 -1 307 val_307 -1 308 val_308 -1 315 val_315 -1 317 val_317 -1 321 val_321 -1 33 val_33 -1 331 val_331 -1 356 val_356 -1 360 val_360 -1 366 val_366 -1 367 val_367 -1 368 val_368 -1 37 val_37 -1 378 val_378 -1 379 val_379 -1 382 val_382 -1 389 val_389 -1 395 val_395 -1 400 val_400 -1 402 val_402 -1 406 val_406 -1 407 val_407 -1 42 val_42 -1 430 val_430 -1 436 val_436 -1 44 val_44 -1 452 val_452 -1 466 val_466 -1 467 val_467 -1 472 val_472 -1 480 val_480 -1 484 val_484 -1 485 val_485 -1 487 val_487 -1 495 val_495 -1 496 val_496 -1 498 val_498 -1 51 val_51 -1 57 val_57 -1 58 val_58 -1 65 val_65 -1 66 val_66 -1 69 val_69 -1 70 val_70 -1 78 val_78 -1 8 val_8 -1 92 val_92 -1 96 val_96 -1 104 val_104 -1 111 val_111 -1 114 val_114 -1 118 val_118 -1 12 val_12 -1 133 val_133 -1 138 val_138 -1 153 val_153 -1 156 val_156 -1 162 val_162 -1 168 val_168 -1 174 val_174 -1 176 val_176 -1 178 val_178 -1 181 val_181 -1 186 val_186 -1 187 val_187 -1 192 val_192 -1 197 val_197 -1 199 val_199 -1 2 val_2 -1 201 val_201 -1 203 val_203 -1 208 val_208 -1 214 val_214 -1 218 val_218 -1 219 val_219 -1 230 val_230 -1 237 val_237 -1 249 val_249 -1 256 val_256 -1 257 val_257 -1 277 val_277 -1 278 val_278 -1 284 val_284 -1 285 val_285 -1 310 val_310 -1 322 val_322 -1 327 val_327 -1 333 val_333 -1 336 val_336 -1 338 val_338 -1 341 val_341 -1 344 val_344 -1 348 val_348 -1 351 val_351 -1 362 val_362 -1 364 val_364 -1 374 val_374 -1 392 val_392 -1 397 val_397 -1 4 val_4 -1 403 val_403 -1 404 val_404 -1 419 val_419 -1 421 val_421 -1 427 val_427 -1 429 val_429 -1 437 val_437 -1 438 val_438 -1 439 val_439 -1 454 val_454 -1 455 val_455 -1 460 val_460 -1 463 val_463 -1 470 val_470 -1 478 val_478 -1 479 val_479 -1 489 val_489 -1 491 val_491 -1 492 val_492 -1 53 val_53 -1 67 val_67 -1 82 val_82 -1 90 val_90 -1 97 val_97 -1 10 val_10 -1 125 val_125 -1 134 val_134 -1 143 val_143 -1 145 val_145 -1 149 val_149 -1 150 val_150 -1 158 val_158 -1 163 val_163 -1 164 val_164 -1 166 val_166 -1 17 val_17 -1 170 val_170 -1 172 val_172 -1 189 val_189 -1 19 val_19 -1 195 val_195 -1 196 val_196 -1 20 val_20 -1 209 val_209 -1 216 val_216 -1 217 val_217 -1 224 val_224 -1 228 val_228 -1 229 val_229 -1 233 val_233 -1 241 val_241 -1 244 val_244 -1 247 val_247 -1 255 val_255 -1 258 val_258 -1 26 val_26 -1 263 val_263 -1 265 val_265 -1 266 val_266 -1 272 val_272 -1 273 val_273 -1 274 val_274 -1 281 val_281 -1 291 val_291 -1 296 val_296 -1 30 val_30 -1 302 val_302 -1 309 val_309 -1 316 val_316 -1 318 val_318 -1 325 val_325 -1 332 val_332 -1 335 val_335 -1 339 val_339 -1 342 val_342 -1 345 val_345 -1 353 val_353 -1 369 val_369 -1 373 val_373 -1 375 val_375 -1 377 val_377 -1 384 val_384 -1 386 val_386 -1 394 val_394 -1 396 val_396 -1 399 val_399 -1 401 val_401 -1 41 val_41 -1 413 val_413 -1 414 val_414 -1 431 val_431 -1 446 val_446 -1 448 val_448 -1 449 val_449 -1 459 val_459 -1 462 val_462 -1 468 val_468 -1 47 val_47 -1 482 val_482 -1 490 val_490 -1 493 val_493 -1 494 val_494 -1 497 val_497 -1 5 val_5 -1 54 val_54 -1 74 val_74 -1 77 val_77 -1 80 val_80 -1 84 val_84 -1 85 val_85 -1 87 val_87 -1 9 val_9 -1 95 val_95 -1 100 val_100 -1 11 val_11 -1 113 val_113 -1 119 val_119 -1 126 val_126 -1 128 val_128 -1 137 val_137 -1 146 val_146 -1 15 val_15 -1 152 val_152 -1 165 val_165 -1 167 val_167 -1 169 val_169 -1 175 val_175 -1 177 val_177 -1 179 val_179 -1 18 val_18 -1 194 val_194 -1 200 val_200 -1 202 val_202 -1 221 val_221 -1 222 val_222 -1 226 val_226 -1 242 val_242 -1 252 val_252 -1 262 val_262 -1 27 val_27 -1 275 val_275 -1 280 val_280 -1 282 val_282 -1 283 val_283 -1 287 val_287 -1 288 val_288 -1 289 val_289 -1 292 val_292 -1 311 val_311 -1 323 val_323 -1 34 val_34 -1 35 val_35 -1 365 val_365 -1 393 val_393 -1 409 val_409 -1 411 val_411 -1 417 val_417 -1 418 val_418 -1 424 val_424 -1 43 val_43 -1 432 val_432 -1 435 val_435 -1 443 val_443 -1 444 val_444 -1 453 val_453 -1 457 val_457 -1 458 val_458 -1 469 val_469 -1 475 val_475 -1 477 val_477 -1 481 val_481 -1 483 val_483 -1 64 val_64 -1 72 val_72 -1 76 val_76 -1 83 val_83 -1 86 val_86 -1 98 val_98 -PREHOOK: query: SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: SELECT -count(1) as counts, -key, -value -FROM -( - -SELECT -a.key, a.value -FROM srcpart a JOIN srcpart b -ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' -AND a.key = b.key - -UNION DISTINCT - -select key, value -FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -WHERE ds='2008-04-08' and hr='11' -) a -group by key, value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -1 0 val_0 -1 103 val_103 -1 105 val_105 -1 116 val_116 -1 120 val_120 -1 129 val_129 -1 131 val_131 -1 136 val_136 -1 155 val_155 -1 157 val_157 -1 160 val_160 -1 180 val_180 -1 183 val_183 -1 190 val_190 -1 191 val_191 -1 193 val_193 -1 205 val_205 -1 207 val_207 -1 213 val_213 -1 223 val_223 -1 235 val_235 -1 238 val_238 -1 239 val_239 -1 24 val_24 -1 248 val_248 -1 260 val_260 -1 28 val_28 -1 286 val_286 -1 298 val_298 -1 305 val_305 -1 306 val_306 -1 307 val_307 -1 308 val_308 -1 315 val_315 -1 317 val_317 -1 321 val_321 -1 33 val_33 -1 331 val_331 -1 356 val_356 -1 360 val_360 -1 366 val_366 -1 367 val_367 -1 368 val_368 -1 37 val_37 -1 378 val_378 -1 379 val_379 -1 382 val_382 -1 389 val_389 -1 395 val_395 -1 400 val_400 -1 402 val_402 -1 406 val_406 -1 407 val_407 -1 42 val_42 -1 430 val_430 -1 436 val_436 -1 44 val_44 -1 452 val_452 -1 466 val_466 -1 467 val_467 -1 472 val_472 -1 480 val_480 -1 484 val_484 -1 485 val_485 -1 487 val_487 -1 495 val_495 -1 496 val_496 -1 498 val_498 -1 51 val_51 -1 57 val_57 -1 58 val_58 -1 65 val_65 -1 66 val_66 -1 69 val_69 -1 70 val_70 -1 78 val_78 -1 8 val_8 -1 92 val_92 -1 96 val_96 -1 104 val_104 -1 111 val_111 -1 114 val_114 -1 118 val_118 -1 12 val_12 -1 133 val_133 -1 138 val_138 -1 153 val_153 -1 156 val_156 -1 162 val_162 -1 168 val_168 -1 174 val_174 -1 176 val_176 -1 178 val_178 -1 181 val_181 -1 186 val_186 -1 187 val_187 -1 192 val_192 -1 197 val_197 -1 199 val_199 -1 2 val_2 -1 201 val_201 -1 203 val_203 -1 208 val_208 -1 214 val_214 -1 218 val_218 -1 219 val_219 -1 230 val_230 -1 237 val_237 -1 249 val_249 -1 256 val_256 -1 257 val_257 -1 277 val_277 -1 278 val_278 -1 284 val_284 -1 285 val_285 -1 310 val_310 -1 322 val_322 -1 327 val_327 -1 333 val_333 -1 336 val_336 -1 338 val_338 -1 341 val_341 -1 344 val_344 -1 348 val_348 -1 351 val_351 -1 362 val_362 -1 364 val_364 -1 374 val_374 -1 392 val_392 -1 397 val_397 -1 4 val_4 -1 403 val_403 -1 404 val_404 -1 419 val_419 -1 421 val_421 -1 427 val_427 -1 429 val_429 -1 437 val_437 -1 438 val_438 -1 439 val_439 -1 454 val_454 -1 455 val_455 -1 460 val_460 -1 463 val_463 -1 470 val_470 -1 478 val_478 -1 479 val_479 -1 489 val_489 -1 491 val_491 -1 492 val_492 -1 53 val_53 -1 67 val_67 -1 82 val_82 -1 90 val_90 -1 97 val_97 -1 10 val_10 -1 125 val_125 -1 134 val_134 -1 143 val_143 -1 145 val_145 -1 149 val_149 -1 150 val_150 -1 158 val_158 -1 163 val_163 -1 164 val_164 -1 166 val_166 -1 17 val_17 -1 170 val_170 -1 172 val_172 -1 189 val_189 -1 19 val_19 -1 195 val_195 -1 196 val_196 -1 20 val_20 -1 209 val_209 -1 216 val_216 -1 217 val_217 -1 224 val_224 -1 228 val_228 -1 229 val_229 -1 233 val_233 -1 241 val_241 -1 244 val_244 -1 247 val_247 -1 255 val_255 -1 258 val_258 -1 26 val_26 -1 263 val_263 -1 265 val_265 -1 266 val_266 -1 272 val_272 -1 273 val_273 -1 274 val_274 -1 281 val_281 -1 291 val_291 -1 296 val_296 -1 30 val_30 -1 302 val_302 -1 309 val_309 -1 316 val_316 -1 318 val_318 -1 325 val_325 -1 332 val_332 -1 335 val_335 -1 339 val_339 -1 342 val_342 -1 345 val_345 -1 353 val_353 -1 369 val_369 -1 373 val_373 -1 375 val_375 -1 377 val_377 -1 384 val_384 -1 386 val_386 -1 394 val_394 -1 396 val_396 -1 399 val_399 -1 401 val_401 -1 41 val_41 -1 413 val_413 -1 414 val_414 -1 431 val_431 -1 446 val_446 -1 448 val_448 -1 449 val_449 -1 459 val_459 -1 462 val_462 -1 468 val_468 -1 47 val_47 -1 482 val_482 -1 490 val_490 -1 493 val_493 -1 494 val_494 -1 497 val_497 -1 5 val_5 -1 54 val_54 -1 74 val_74 -1 77 val_77 -1 80 val_80 -1 84 val_84 -1 85 val_85 -1 87 val_87 -1 9 val_9 -1 95 val_95 -1 100 val_100 -1 11 val_11 -1 113 val_113 -1 119 val_119 -1 126 val_126 -1 128 val_128 -1 137 val_137 -1 146 val_146 -1 15 val_15 -1 152 val_152 -1 165 val_165 -1 167 val_167 -1 169 val_169 -1 175 val_175 -1 177 val_177 -1 179 val_179 -1 18 val_18 -1 194 val_194 -1 200 val_200 -1 202 val_202 -1 221 val_221 -1 222 val_222 -1 226 val_226 -1 242 val_242 -1 252 val_252 -1 262 val_262 -1 27 val_27 -1 275 val_275 -1 280 val_280 -1 282 val_282 -1 283 val_283 -1 287 val_287 -1 288 val_288 -1 289 val_289 -1 292 val_292 -1 311 val_311 -1 323 val_323 -1 34 val_34 -1 35 val_35 -1 365 val_365 -1 393 val_393 -1 409 val_409 -1 411 val_411 -1 417 val_417 -1 418 val_418 -1 424 val_424 -1 43 val_43 -1 432 val_432 -1 435 val_435 -1 443 val_443 -1 444 val_444 -1 453 val_453 -1 457 val_457 -1 458 val_458 -1 469 val_469 -1 475 val_475 -1 477 val_477 -1 481 val_481 -1 483 val_483 -1 64 val_64 -1 72 val_72 -1 76 val_76 -1 83 val_83 -1 86 val_86 -1 98 val_98 -PREHOOK: query: create table jackson_sev_same as select * from src -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@jackson_sev_same -POSTHOOK: query: create table jackson_sev_same as select * from src -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@jackson_sev_same -POSTHOOK: Lineage: jackson_sev_same.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: jackson_sev_same.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table dim_pho as select * from src -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src +0 val_0 1 +10 val_10 1 +100 val_100 1 +103 val_103 1 +104 val_104 1 +105 val_105 1 +11 val_11 1 +111 val_111 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +119 val_119 1 +12 val_12 1 +120 val_120 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +149 val_149 1 +15 val_15 1 +150 val_150 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +17 val_17 1 +170 val_170 1 +172 val_172 1 +174 val_174 1 +175 val_175 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +18 val_18 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +189 val_189 1 +19 val_19 1 +190 val_190 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +199 val_199 1 +2 val_2 1 +20 val_20 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +205 val_205 1 +207 val_207 1 +208 val_208 1 +209 val_209 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +230 val_230 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +238 val_238 1 +239 val_239 1 +24 val_24 1 +241 val_241 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +26 val_26 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +266 val_266 1 +27 val_27 1 +272 val_272 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +278 val_278 1 +28 val_28 1 +280 val_280 1 +281 val_281 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +30 val_30 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +317 val_317 1 +318 val_318 1 +321 val_321 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +327 val_327 1 +33 val_33 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +34 val_34 1 +341 val_341 1 +342 val_342 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +35 val_35 1 +351 val_351 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +37 val_37 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +396 val_396 1 +397 val_397 1 +399 val_399 1 +4 val_4 1 +400 val_400 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +404 val_404 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +41 val_41 1 +411 val_411 1 +413 val_413 1 +414 val_414 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +42 val_42 1 +421 val_421 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +43 val_43 1 +430 val_430 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +439 val_439 1 +44 val_44 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +463 val_463 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +469 val_469 1 +47 val_47 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +5 val_5 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +8 val_8 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +9 val_9 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +98 val_98 1 +tst1 500 1 +PREHOOK: query: CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@dim_pho -POSTHOOK: query: create table dim_pho as select * from src -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src +PREHOOK: Output: default@DEST118 +POSTHOOK: query: CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@dim_pho -POSTHOOK: Lineage: dim_pho.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dim_pho.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table jackson_sev_add as select * from src -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src +POSTHOOK: Output: default@DEST118 +PREHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@jackson_sev_add -POSTHOOK: query: create table jackson_sev_add as select * from src -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src +PREHOOK: Output: default@DEST218 +POSTHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@jackson_sev_add -POSTHOOK: Lineage: jackson_sev_add.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: jackson_sev_add.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +POSTHOOK: Output: default@DEST218 +PREHOOK: query: explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value PREHOOK: type: QUERY -POSTHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +POSTHOOK: query: explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-2 Tez #### A masked pattern was here #### Edges: - Map 3 <- Union 4 (CONTAINS) - Map 6 <- Union 4 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: a - Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(key) = 97.0) (type: boolean) - Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: dim_pho - Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(key) = 97.0) (type: boolean) - Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 Map Operator Tree: TableScan - alias: jackson_sev_add - Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(key) = 97.0) (type: boolean) - Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 - Execution mode: llap Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 275 Data size: 48171 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), _col2 (type: string) + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 48171 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 48171 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 Execution mode: llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Union 4 - Vertex: Union 4 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest118 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest218 + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest118 -PREHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 + Stage: Stage-4 + Stats Work + Basic Stats Work: + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest218 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest118 +PREHOOK: Output: default@dest218 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest118 +POSTHOOK: Output: default@dest218 +POSTHOOK: Lineage: dest118.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest118.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest118 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest118 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +tst1 500 +PREHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 PREHOOK: type: QUERY -PREHOOK: Input: default@dim_pho -PREHOOK: Input: default@jackson_sev_add -PREHOOK: Input: default@jackson_sev_same +PREHOOK: Input: default@dest218 #### A masked pattern was here #### -POSTHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +POSTHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 POSTHOOK: type: QUERY -POSTHOOK: Input: default@dim_pho -POSTHOOK: Input: default@jackson_sev_add -POSTHOOK: Input: default@jackson_sev_same +POSTHOOK: Input: default@dest218 #### A masked pattern was here #### -97 val_97 -97 val_97 -PREHOOK: query: create table union_subq_union(key int, value string) +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +103 val_103 val_103 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +119 val_119 val_119 +12 val_12 val_12 +120 val_120 val_120 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +149 val_149 val_149 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +174 val_174 val_174 +175 val_175 val_175 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +205 val_205 val_205 +207 val_207 val_207 +208 val_208 val_208 +209 val_209 val_209 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +230 val_230 val_230 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +238 val_238 val_238 +239 val_239 val_239 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +281 val_281 val_281 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +317 val_317 val_317 +318 val_318 val_318 +321 val_321 val_321 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +396 val_396 val_396 +397 val_397 val_397 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +404 val_404 val_404 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +414 val_414 val_414 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +463 val_463 val_463 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +5 val_5 val_5 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +98 val_98 val_98 +tst1 500 500 +PREHOOK: query: CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@union_subq_union -POSTHOOK: query: create table union_subq_union(key int, value string) +PREHOOK: Output: default@DEST119 +POSTHOOK: query: CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@union_subq_union -PREHOOK: query: explain -insert overwrite table union_subq_union -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a -PREHOOK: type: QUERY -POSTHOOK: query: explain -insert overwrite table union_subq_union -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a +POSTHOOK: Output: default@DEST119 +PREHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST219 +POSTHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST219 +PREHOOK: query: explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-2 Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 6 (CONTAINS) + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - keys: key (type: string), value (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 Map Operator Tree: TableScan - alias: src + alias: s2 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: key, value + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: count() - keys: key (type: string), value (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - Reducer 5 - Execution mode: llap + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 Execution mode: llap Reduce Operator Tree: Group By Operator - aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - mode: hash + aggregations: count(_col1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest119 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 6 - Vertex: Union 6 + Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 114000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest219 + Union 3 + Vertex: Union 3 - Stage: Stage-2 + Stage: Stage-3 Dependency Collection Stage: Stage-0 @@ -11032,103 +2718,756 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + name: default.dest119 - Stage: Stage-3 + Stage: Stage-4 Stats Work Basic Stats Work: -PREHOOK: query: insert overwrite table union_subq_union -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest219 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest119 +PREHOOK: Output: default@dest219 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest119 +POSTHOOK: Output: default@dest219 +POSTHOOK: Lineage: dest119.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest119.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest119 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest119 +#### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +tst1 1 +PREHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest219 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest219 +#### A masked pattern was here #### +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +103 val_103 val_103 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +119 val_119 val_119 +12 val_12 val_12 +120 val_120 val_120 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +149 val_149 val_149 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +174 val_174 val_174 +175 val_175 val_175 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +205 val_205 val_205 +207 val_207 val_207 +208 val_208 val_208 +209 val_209 val_209 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +230 val_230 val_230 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +238 val_238 val_238 +239 val_239 val_239 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +281 val_281 val_281 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +317 val_317 val_317 +318 val_318 val_318 +321 val_321 val_321 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +396 val_396 val_396 +397 val_397 val_397 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +404 val_404 val_404 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +414 val_414 val_414 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +463 val_463 val_463 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +5 val_5 val_5 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +98 val_98 val_98 +tst1 500 500 +PREHOOK: query: create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dst_union22 +POSTHOOK: query: create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dst_union22 +PREHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dst_union22_delta +POSTHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dst_union22_delta +PREHOOK: query: insert overwrite table dst_union22 partition (ds='1') +select key, value, key , value from src PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union -POSTHOOK: query: insert overwrite table union_subq_union -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a +PREHOOK: Output: default@dst_union22@ds=1 +POSTHOOK: query: insert overwrite table dst_union22 partition (ds='1') +select key, value, key , value from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union -POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from union_subq_union order by key, value limit 20 +POSTHOOK: Output: default@dst_union22@ds=1 +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') +select key, key, value, key, value, value from src PREHOOK: type: QUERY -PREHOOK: Input: default@union_subq_union -#### A masked pattern was here #### -POSTHOOK: query: select * from union_subq_union order by key, value limit 20 +PREHOOK: Input: default@src +PREHOOK: Output: default@dst_union22_delta@ds=1 +POSTHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') +select key, key, value, key, value, value from src POSTHOOK: type: QUERY -POSTHOOK: Input: default@union_subq_union -#### A masked pattern was here #### -0 val_0 -2 val_2 -4 val_4 -5 val_5 -8 val_8 -9 val_9 -10 val_10 -11 val_11 -12 val_12 -15 val_15 -17 val_17 -18 val_18 -19 val_19 -20 val_20 -24 val_24 -26 val_26 -27 val_27 -28 val_28 -30 val_30 -33 val_33 -PREHOOK: query: create table union_subq_union29(key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@union_subq_union29 -POSTHOOK: query: create table union_subq_union29(key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@union_subq_union29 -PREHOOK: query: explain -insert overwrite table union_subq_union29 -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value from src - UNION DISTINCT - select key, value from src - ) subq -) a +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dst_union22_delta@ds=1 +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain extended +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq PREHOOK: type: QUERY -POSTHOOK: query: explain -insert overwrite table union_subq_union29 -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value from src - UNION DISTINCT - select key, value from src - ) subq -) a +POSTHOOK: query: explain extended +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -11142,418 +3481,812 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union29 - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 - - Stage: Stage-2 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union29 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - -PREHOOK: query: insert overwrite table union_subq_union29 -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value from src - UNION DISTINCT - select key, value from src - ) subq -) a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union29 -POSTHOOK: query: insert overwrite table union_subq_union29 -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value from src - UNION DISTINCT - select key, value from src - ) subq -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union29 -POSTHOOK: Lineage: union_subq_union29.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: union_subq_union29.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from union_subq_union29 order by key, value limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@union_subq_union29 + alias: dst_union22_delta + Statistics: Num rows: 500 Data size: 454856 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(k0) <= 50.0) (type: boolean) + Statistics: Num rows: 166 Data size: 151012 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 166 Data size: 151012 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + null sort order: aaaa + sort order: ++++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: #### A masked pattern was here #### -POSTHOOK: query: select * from union_subq_union29 order by key, value limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@union_subq_union29 + Path -> Partition: #### A masked pattern was here #### -0 val_0 -2 val_2 -4 val_4 -5 val_5 -8 val_8 -9 val_9 -10 val_10 -11 val_11 -12 val_12 -15 val_15 -17 val_17 -18 val_18 -19 val_19 -20 val_20 -24 val_24 -26 val_26 -27 val_27 -28 val_28 -30 val_30 -33 val_33 -PREHOOK: query: explain -SELECT * -FROM ( - SELECT 1 AS id - FROM (SELECT * FROM src LIMIT 1) s1 - UNION DISTINCT - SELECT 2 AS id - FROM (SELECT * FROM src LIMIT 1) s1 - UNION DISTINCT - SELECT 3 AS id - FROM (SELECT * FROM src LIMIT 1) s2 - UNION DISTINCT - SELECT 4 AS id - FROM (SELECT * FROM src LIMIT 1) s2 - CLUSTER BY id -) a -PREHOOK: type: QUERY -POSTHOOK: query: explain -SELECT * -FROM ( - SELECT 1 AS id - FROM (SELECT * FROM src LIMIT 1) s1 - UNION DISTINCT - SELECT 2 AS id - FROM (SELECT * FROM src LIMIT 1) s1 - UNION DISTINCT - SELECT 3 AS id - FROM (SELECT * FROM src LIMIT 1) s2 - UNION DISTINCT - SELECT 4 AS id - FROM (SELECT * FROM src LIMIT 1) s2 - CLUSTER BY id -) a -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string #### A masked pattern was here #### - Edges: - Reducer 10 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) - Reducer 9 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 #### A masked pattern was here #### - Vertices: - Map 1 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [dst_union22_delta] + Map 4 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + alias: a + Statistics: Num rows: 500 Data size: 278292 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(k1) > 20.0) (type: boolean) + Statistics: Num rows: 166 Data size: 89056 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: k1 (type: string), k2 (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 166 Data size: 89056 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 11 + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 89056 Basic stats: COMPLETE Column stats: PARTIAL + tag: 0 + value expressions: _col1 (type: string), _col2 (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 11124 + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11624 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + name: default.dst_union22 + Truncated Path -> Alias: + /dst_union22/ds=1 [a] + Map 6 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + alias: dst_union22_delta + Statistics: Num rows: 500 Data size: 367272 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((UDFToDouble(k0) > 50.0) and (UDFToDouble(k1) > 20.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 40399 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 40399 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 4 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 2 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Execution mode: llap + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 40399 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col1 (type: string), _col2 (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [dst_union22_delta] + Reducer 3 + Needs Tagging: false Reduce Operator Tree: - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 174 Data size: 124486 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ + Statistics: Num rows: 174 Data size: 124486 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Reducer 5 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + filter mappings: + 0 [1, 1] + filter predicates: + 0 {(VALUE._col1 = '1')} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 182 Data size: 97961 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 97961 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: int) + keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + null sort order: aaaa + sort order: ++++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 348 Data size: 248973 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2 + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + + Stage: Stage-3 + Stats Work + Basic Stats Work: +#### A masked pattern was here #### + +PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +PREHOOK: type: QUERY +PREHOOK: Input: default@dst_union22 +PREHOOK: Input: default@dst_union22@ds=1 +PREHOOK: Input: default@dst_union22_delta +PREHOOK: Input: default@dst_union22_delta@ds=1 +PREHOOK: Output: default@dst_union22@ds=2 +POSTHOOK: query: insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dst_union22 +POSTHOOK: Input: default@dst_union22@ds=1 +POSTHOOK: Input: default@dst_union22_delta +POSTHOOK: Input: default@dst_union22_delta@ds=1 +POSTHOOK: Output: default@dst_union22@ds=2 +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k1 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k1, type:string, comment:null), (dst_union22)a.FieldSchema(name:k1, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k2 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k2, type:string, comment:null), (dst_union22)a.FieldSchema(name:k2, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k3 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k4 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), ] +PREHOOK: query: select * from dst_union22 where ds = '2' +PREHOOK: type: QUERY +PREHOOK: Input: default@dst_union22 +PREHOOK: Input: default@dst_union22@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: select * from dst_union22 where ds = '2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dst_union22 +POSTHOOK: Input: default@dst_union22@ds=2 +#### A masked pattern was here #### +0 val_0 0 val_0 2 +10 val_10 10 val_10 2 +100 val_100 100 val_100 2 +103 val_103 103 val_103 2 +104 val_104 104 val_104 2 +105 val_105 105 val_105 2 +11 val_11 11 val_11 2 +111 val_111 111 val_111 2 +113 val_113 113 val_113 2 +114 val_114 114 val_114 2 +116 val_116 116 val_116 2 +118 val_118 118 val_118 2 +119 val_119 119 val_119 2 +12 val_12 12 val_12 2 +120 val_120 120 val_120 2 +125 val_125 125 val_125 2 +126 val_126 126 val_126 2 +128 val_128 128 val_128 2 +129 val_129 129 val_129 2 +131 val_131 131 val_131 2 +133 val_133 133 val_133 2 +134 val_134 134 val_134 2 +136 val_136 136 val_136 2 +137 val_137 137 val_137 2 +138 val_138 138 val_138 2 +143 val_143 143 val_143 2 +145 val_145 145 val_145 2 +146 val_146 146 val_146 2 +149 val_149 149 val_149 2 +15 val_15 15 val_15 2 +150 val_150 150 val_150 2 +152 val_152 152 val_152 2 +153 val_153 153 val_153 2 +155 val_155 155 val_155 2 +156 val_156 156 val_156 2 +157 val_157 157 val_157 2 +158 val_158 158 val_158 2 +160 val_160 160 val_160 2 +162 val_162 162 val_162 2 +163 val_163 163 val_163 2 +164 val_164 164 val_164 2 +165 val_165 165 val_165 2 +166 val_166 166 val_166 2 +167 val_167 167 val_167 2 +168 val_168 168 val_168 2 +169 val_169 169 val_169 2 +17 val_17 17 val_17 2 +170 val_170 170 val_170 2 +172 val_172 172 val_172 2 +174 val_174 174 val_174 2 +175 val_175 175 val_175 2 +176 val_176 176 val_176 2 +177 val_177 177 val_177 2 +178 val_178 178 val_178 2 +179 val_179 179 val_179 2 +18 val_18 18 val_18 2 +180 val_180 180 val_180 2 +181 val_181 181 val_181 2 +183 val_183 183 val_183 2 +186 val_186 186 val_186 2 +187 val_187 187 val_187 2 +189 val_189 189 val_189 2 +19 val_19 19 val_19 2 +190 val_190 190 val_190 2 +191 val_191 191 val_191 2 +192 val_192 192 val_192 2 +193 val_193 193 val_193 2 +194 val_194 194 val_194 2 +195 val_195 195 val_195 2 +196 val_196 196 val_196 2 +197 val_197 197 val_197 2 +199 val_199 199 val_199 2 +2 val_2 2 val_2 2 +20 val_20 20 val_20 2 +200 val_200 200 val_200 2 +201 val_201 201 val_201 2 +202 val_202 202 val_202 2 +203 val_203 203 val_203 2 +205 val_205 205 val_205 2 +207 val_207 207 val_207 2 +208 val_208 208 val_208 2 +209 val_209 209 val_209 2 +213 val_213 213 val_213 2 +214 val_214 214 val_214 2 +216 val_216 216 val_216 2 +217 val_217 217 val_217 2 +218 val_218 218 val_218 2 +219 val_219 219 val_219 2 +221 val_221 221 val_221 2 +222 val_222 222 val_222 2 +223 val_223 223 val_223 2 +224 val_224 224 val_224 2 +226 val_226 226 val_226 2 +228 val_228 228 val_228 2 +229 val_229 229 val_229 2 +230 val_230 230 val_230 2 +233 val_233 233 val_233 2 +235 val_235 235 val_235 2 +237 val_237 237 val_237 2 +238 val_238 238 val_238 2 +239 val_239 239 val_239 2 +24 val_24 NULL NULL 2 +24 val_24 24 val_24 2 +241 val_241 241 val_241 2 +242 val_242 242 val_242 2 +244 val_244 244 val_244 2 +247 val_247 247 val_247 2 +248 val_248 248 val_248 2 +249 val_249 249 val_249 2 +252 val_252 252 val_252 2 +255 val_255 255 val_255 2 +256 val_256 256 val_256 2 +257 val_257 257 val_257 2 +258 val_258 258 val_258 2 +26 val_26 NULL NULL 2 +26 val_26 26 val_26 2 +260 val_260 260 val_260 2 +262 val_262 262 val_262 2 +263 val_263 263 val_263 2 +265 val_265 265 val_265 2 +266 val_266 266 val_266 2 +27 val_27 NULL NULL 2 +27 val_27 27 val_27 2 +272 val_272 272 val_272 2 +273 val_273 273 val_273 2 +274 val_274 274 val_274 2 +275 val_275 275 val_275 2 +277 val_277 277 val_277 2 +278 val_278 278 val_278 2 +28 val_28 NULL NULL 2 +28 val_28 28 val_28 2 +280 val_280 280 val_280 2 +281 val_281 281 val_281 2 +282 val_282 282 val_282 2 +283 val_283 283 val_283 2 +284 val_284 284 val_284 2 +285 val_285 285 val_285 2 +286 val_286 286 val_286 2 +287 val_287 287 val_287 2 +288 val_288 288 val_288 2 +289 val_289 289 val_289 2 +291 val_291 291 val_291 2 +292 val_292 292 val_292 2 +296 val_296 296 val_296 2 +298 val_298 298 val_298 2 +30 val_30 NULL NULL 2 +30 val_30 30 val_30 2 +302 val_302 302 val_302 2 +305 val_305 305 val_305 2 +306 val_306 306 val_306 2 +307 val_307 307 val_307 2 +308 val_308 308 val_308 2 +309 val_309 309 val_309 2 +310 val_310 310 val_310 2 +311 val_311 311 val_311 2 +315 val_315 315 val_315 2 +316 val_316 316 val_316 2 +317 val_317 317 val_317 2 +318 val_318 318 val_318 2 +321 val_321 321 val_321 2 +322 val_322 322 val_322 2 +323 val_323 323 val_323 2 +325 val_325 325 val_325 2 +327 val_327 327 val_327 2 +33 val_33 NULL NULL 2 +33 val_33 33 val_33 2 +331 val_331 331 val_331 2 +332 val_332 332 val_332 2 +333 val_333 333 val_333 2 +335 val_335 335 val_335 2 +336 val_336 336 val_336 2 +338 val_338 338 val_338 2 +339 val_339 339 val_339 2 +34 val_34 NULL NULL 2 +34 val_34 34 val_34 2 +341 val_341 341 val_341 2 +342 val_342 342 val_342 2 +344 val_344 344 val_344 2 +345 val_345 345 val_345 2 +348 val_348 348 val_348 2 +35 val_35 NULL NULL 2 +35 val_35 35 val_35 2 +351 val_351 351 val_351 2 +353 val_353 353 val_353 2 +356 val_356 356 val_356 2 +360 val_360 360 val_360 2 +362 val_362 362 val_362 2 +364 val_364 364 val_364 2 +365 val_365 365 val_365 2 +366 val_366 366 val_366 2 +367 val_367 367 val_367 2 +368 val_368 368 val_368 2 +369 val_369 369 val_369 2 +37 val_37 NULL NULL 2 +37 val_37 37 val_37 2 +373 val_373 373 val_373 2 +374 val_374 374 val_374 2 +375 val_375 375 val_375 2 +377 val_377 377 val_377 2 +378 val_378 378 val_378 2 +379 val_379 379 val_379 2 +382 val_382 382 val_382 2 +384 val_384 384 val_384 2 +386 val_386 386 val_386 2 +389 val_389 389 val_389 2 +392 val_392 392 val_392 2 +393 val_393 393 val_393 2 +394 val_394 394 val_394 2 +395 val_395 395 val_395 2 +396 val_396 396 val_396 2 +397 val_397 397 val_397 2 +399 val_399 399 val_399 2 +4 val_4 4 val_4 2 +400 val_400 400 val_400 2 +401 val_401 401 val_401 2 +402 val_402 402 val_402 2 +403 val_403 403 val_403 2 +404 val_404 404 val_404 2 +406 val_406 406 val_406 2 +407 val_407 407 val_407 2 +409 val_409 409 val_409 2 +41 val_41 NULL NULL 2 +41 val_41 41 val_41 2 +411 val_411 411 val_411 2 +413 val_413 413 val_413 2 +414 val_414 414 val_414 2 +417 val_417 417 val_417 2 +418 val_418 418 val_418 2 +419 val_419 419 val_419 2 +42 val_42 NULL NULL 2 +42 val_42 42 val_42 2 +421 val_421 421 val_421 2 +424 val_424 424 val_424 2 +427 val_427 427 val_427 2 +429 val_429 429 val_429 2 +43 val_43 NULL NULL 2 +43 val_43 43 val_43 2 +430 val_430 430 val_430 2 +431 val_431 431 val_431 2 +432 val_432 432 val_432 2 +435 val_435 435 val_435 2 +436 val_436 436 val_436 2 +437 val_437 437 val_437 2 +438 val_438 438 val_438 2 +439 val_439 439 val_439 2 +44 val_44 NULL NULL 2 +44 val_44 44 val_44 2 +443 val_443 443 val_443 2 +444 val_444 444 val_444 2 +446 val_446 446 val_446 2 +448 val_448 448 val_448 2 +449 val_449 449 val_449 2 +452 val_452 452 val_452 2 +453 val_453 453 val_453 2 +454 val_454 454 val_454 2 +455 val_455 455 val_455 2 +457 val_457 457 val_457 2 +458 val_458 458 val_458 2 +459 val_459 459 val_459 2 +460 val_460 460 val_460 2 +462 val_462 462 val_462 2 +463 val_463 463 val_463 2 +466 val_466 466 val_466 2 +467 val_467 467 val_467 2 +468 val_468 468 val_468 2 +469 val_469 469 val_469 2 +47 val_47 NULL NULL 2 +47 val_47 47 val_47 2 +470 val_470 470 val_470 2 +472 val_472 472 val_472 2 +475 val_475 475 val_475 2 +477 val_477 477 val_477 2 +478 val_478 478 val_478 2 +479 val_479 479 val_479 2 +480 val_480 480 val_480 2 +481 val_481 481 val_481 2 +482 val_482 482 val_482 2 +483 val_483 483 val_483 2 +484 val_484 484 val_484 2 +485 val_485 485 val_485 2 +487 val_487 487 val_487 2 +489 val_489 489 val_489 2 +490 val_490 490 val_490 2 +491 val_491 491 val_491 2 +492 val_492 492 val_492 2 +493 val_493 493 val_493 2 +494 val_494 494 val_494 2 +495 val_495 495 val_495 2 +496 val_496 496 val_496 2 +497 val_497 497 val_497 2 +498 val_498 498 val_498 2 +5 val_5 5 val_5 2 +51 val_51 51 val_51 2 +53 val_53 53 val_53 2 +54 val_54 54 val_54 2 +57 val_57 57 val_57 2 +58 val_58 58 val_58 2 +64 val_64 64 val_64 2 +65 val_65 65 val_65 2 +66 val_66 66 val_66 2 +67 val_67 67 val_67 2 +69 val_69 69 val_69 2 +70 val_70 70 val_70 2 +72 val_72 72 val_72 2 +74 val_74 74 val_74 2 +76 val_76 76 val_76 2 +77 val_77 77 val_77 2 +78 val_78 78 val_78 2 +8 val_8 8 val_8 2 +80 val_80 80 val_80 2 +82 val_82 82 val_82 2 +83 val_83 83 val_83 2 +84 val_84 84 val_84 2 +85 val_85 85 val_85 2 +86 val_86 86 val_86 2 +87 val_87 87 val_87 2 +9 val_9 9 val_9 2 +90 val_90 90 val_90 2 +92 val_92 92 val_92 2 +95 val_95 95 val_95 2 +96 val_96 96 val_96 2 +97 val_97 97 val_97 2 +98 val_98 98 val_98 2 +PREHOOK: query: explain +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +PREHOOK: type: QUERY +POSTHOOK: query: explain +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: int) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 3 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 - Union 7 - Vertex: Union 7 + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -11561,1368 +4294,2078 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: CREATE TABLE union_out (id int) -PREHOOK: type: CREATETABLE +PREHOOK: query: select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +103 val_103 +105 val_105 +116 val_116 +120 val_120 +129 val_129 +131 val_131 +136 val_136 +155 val_155 +157 val_157 +160 val_160 +180 val_180 +183 val_183 +190 val_190 +191 val_191 +193 val_193 +205 val_205 +207 val_207 +213 val_213 +223 val_223 +235 val_235 +238 val_238 +239 val_239 +24 val_24 +248 val_248 +260 val_260 +28 val_28 +286 val_286 +298 val_298 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +315 val_315 +317 val_317 +321 val_321 +33 val_33 +331 val_331 +356 val_356 +360 val_360 +366 val_366 +367 val_367 +368 val_368 +37 val_37 +378 val_378 +379 val_379 +382 val_382 +389 val_389 +395 val_395 +400 val_400 +402 val_402 +406 val_406 +407 val_407 +42 val_42 +430 val_430 +436 val_436 +44 val_44 +452 val_452 +466 val_466 +467 val_467 +472 val_472 +480 val_480 +484 val_484 +485 val_485 +487 val_487 +495 val_495 +496 val_496 +498 val_498 +51 val_51 +57 val_57 +58 val_58 +65 val_65 +66 val_66 +69 val_69 +70 val_70 +78 val_78 +8 val_8 +92 val_92 +96 val_96 +104 val_104 +111 val_111 +114 val_114 +118 val_118 +12 val_12 +133 val_133 +138 val_138 +153 val_153 +156 val_156 +162 val_162 +168 val_168 +174 val_174 +176 val_176 +178 val_178 +181 val_181 +186 val_186 +187 val_187 +192 val_192 +197 val_197 +199 val_199 +2 val_2 +201 val_201 +203 val_203 +208 val_208 +214 val_214 +218 val_218 +219 val_219 +230 val_230 +237 val_237 +249 val_249 +256 val_256 +257 val_257 +277 val_277 +278 val_278 +284 val_284 +285 val_285 +310 val_310 +322 val_322 +327 val_327 +333 val_333 +336 val_336 +338 val_338 +341 val_341 +344 val_344 +348 val_348 +351 val_351 +362 val_362 +364 val_364 +374 val_374 +392 val_392 +397 val_397 +4 val_4 +403 val_403 +404 val_404 +419 val_419 +421 val_421 +427 val_427 +429 val_429 +437 val_437 +438 val_438 +439 val_439 +454 val_454 +455 val_455 +460 val_460 +463 val_463 +470 val_470 +478 val_478 +479 val_479 +489 val_489 +491 val_491 +492 val_492 +53 val_53 +67 val_67 +82 val_82 +90 val_90 +97 val_97 +10 val_10 +125 val_125 +134 val_134 +143 val_143 +145 val_145 +149 val_149 +150 val_150 +158 val_158 +163 val_163 +164 val_164 +166 val_166 +17 val_17 +170 val_170 +172 val_172 +189 val_189 +19 val_19 +195 val_195 +196 val_196 +20 val_20 +209 val_209 +216 val_216 +217 val_217 +224 val_224 +228 val_228 +229 val_229 +233 val_233 +241 val_241 +244 val_244 +247 val_247 +255 val_255 +258 val_258 +26 val_26 +263 val_263 +265 val_265 +266 val_266 +272 val_272 +273 val_273 +274 val_274 +281 val_281 +291 val_291 +296 val_296 +30 val_30 +302 val_302 +309 val_309 +316 val_316 +318 val_318 +325 val_325 +332 val_332 +335 val_335 +339 val_339 +342 val_342 +345 val_345 +353 val_353 +369 val_369 +373 val_373 +375 val_375 +377 val_377 +384 val_384 +386 val_386 +394 val_394 +396 val_396 +399 val_399 +401 val_401 +41 val_41 +413 val_413 +414 val_414 +431 val_431 +446 val_446 +448 val_448 +449 val_449 +459 val_459 +462 val_462 +468 val_468 +47 val_47 +482 val_482 +490 val_490 +493 val_493 +494 val_494 +497 val_497 +5 val_5 +54 val_54 +74 val_74 +77 val_77 +80 val_80 +84 val_84 +85 val_85 +87 val_87 +9 val_9 +95 val_95 +100 val_100 +11 val_11 +113 val_113 +119 val_119 +126 val_126 +128 val_128 +137 val_137 +146 val_146 +15 val_15 +152 val_152 +165 val_165 +167 val_167 +169 val_169 +175 val_175 +177 val_177 +179 val_179 +18 val_18 +194 val_194 +200 val_200 +202 val_202 +221 val_221 +222 val_222 +226 val_226 +242 val_242 +252 val_252 +262 val_262 +27 val_27 +275 val_275 +280 val_280 +282 val_282 +283 val_283 +287 val_287 +288 val_288 +289 val_289 +292 val_292 +311 val_311 +323 val_323 +34 val_34 +35 val_35 +365 val_365 +393 val_393 +409 val_409 +411 val_411 +417 val_417 +418 val_418 +424 val_424 +43 val_43 +432 val_432 +435 val_435 +443 val_443 +444 val_444 +453 val_453 +457 val_457 +458 val_458 +469 val_469 +475 val_475 +477 val_477 +481 val_481 +483 val_483 +64 val_64 +72 val_72 +76 val_76 +83 val_83 +86 val_86 +98 val_98 +PREHOOK: query: create table src2 as select key, count(1) as count from src group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src PREHOOK: Output: database:default -PREHOOK: Output: default@union_out -POSTHOOK: query: CREATE TABLE union_out (id int) -POSTHOOK: type: CREATETABLE +PREHOOK: Output: default@src2 +POSTHOOK: query: create table src2 as select key, count(1) as count from src group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src POSTHOOK: Output: database:default -POSTHOOK: Output: default@union_out -PREHOOK: query: insert overwrite table union_out -SELECT * -FROM ( - SELECT 1 AS id - FROM (SELECT * FROM src LIMIT 1) s1 +POSTHOOK: Output: default@src2 +POSTHOOK: Lineage: src2.count EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: src2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: create table src3 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src3 +POSTHOOK: query: create table src3 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src3 +POSTHOOK: Lineage: src3.count SIMPLE [(src2)src2.FieldSchema(name:count, type:bigint, comment:null), ] +POSTHOOK: Lineage: src3.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: create table src4 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src4 +POSTHOOK: query: create table src4 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src4 +POSTHOOK: Lineage: src4.count SIMPLE [(src2)src2.FieldSchema(name:count, type:bigint, comment:null), ] +POSTHOOK: Lineage: src4.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: create table src5 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src5 +POSTHOOK: query: create table src5 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src5 +POSTHOOK: Lineage: src5.count SIMPLE [(src2)src2.FieldSchema(name:count, type:bigint, comment:null), ] +POSTHOOK: Lineage: src5.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 UNION DISTINCT - SELECT 2 AS id - FROM (SELECT * FROM src LIMIT 1) s1 + select key, count from src3 where key < 10 UNION DISTINCT - SELECT 3 AS id - FROM (SELECT * FROM src LIMIT 1) s2 + select key, count from src4 where key < 10 UNION DISTINCT - SELECT 4 AS id - FROM (SELECT * FROM src LIMIT 1) s2 - CLUSTER BY id -) a + select key, count(1) as count from src5 where key < 10 group by key +)s PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_out -POSTHOOK: query: insert overwrite table union_out -SELECT * -FROM ( - SELECT 1 AS id - FROM (SELECT * FROM src LIMIT 1) s1 +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 UNION DISTINCT - SELECT 2 AS id - FROM (SELECT * FROM src LIMIT 1) s1 + select key, count from src3 where key < 10 UNION DISTINCT - SELECT 3 AS id - FROM (SELECT * FROM src LIMIT 1) s2 + select key, count from src4 where key < 10 UNION DISTINCT - SELECT 4 AS id - FROM (SELECT * FROM src LIMIT 1) s2 - CLUSTER BY id -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_out -POSTHOOK: Lineage: union_out.id EXPRESSION [] -PREHOOK: query: select * from union_out -PREHOOK: type: QUERY -PREHOOK: Input: default@union_out -#### A masked pattern was here #### -POSTHOOK: query: select * from union_out -POSTHOOK: type: QUERY -POSTHOOK: Input: default@union_out -#### A masked pattern was here #### -4 -1 -2 -3 -PREHOOK: query: create table union_subq_union30(key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@union_subq_union30 -POSTHOOK: query: create table union_subq_union30(key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@union_subq_union30 -PREHOOK: query: explain -insert overwrite table union_subq_union30 -select * from ( - -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a - -UNION DISTINCT - -select key, value from src -) aa -PREHOOK: type: QUERY -POSTHOOK: query: explain -insert overwrite table union_subq_union30 -select * from ( - -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a - -UNION DISTINCT - -select key, value from src -) aa + select key, count(1) as count from src5 where key < 10 group by key +)s POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez #### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 12 <- Union 4 (CONTAINS) - Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 9 <- Union 2 (CONTAINS), Union 8 (SIMPLE_EDGE) + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 8 <- Union 2 (CONTAINS) + Map 9 <- Union 4 (CONTAINS) + Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 + Truncated Path -> Alias: + /src2 [src2] + Map 10 + Map Operator Tree: + TableScan + alias: src5 + Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 #### A masked pattern was here #### - Vertices: - Map 1 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [src5] + Map 8 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash + alias: src3 + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 12 + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src3 [src3] + Map 9 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash + alias: src4 + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [src4] Reducer 11 - Execution mode: llap + Needs Tagging: false Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 51 Data size: 8958 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true Reducer 3 - Execution mode: llap + Needs Tagging: false Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true Reducer 5 - Execution mode: llap + Needs Tagging: false Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union30 - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 8 - Vertex: Union 8 - - Stage: Stage-2 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union30 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - -PREHOOK: query: insert overwrite table union_subq_union30 -select * from ( - -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a - -UNION DISTINCT - -select key, value from src -) aa -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union30 -POSTHOOK: query: insert overwrite table union_subq_union30 -select * from ( - -select * from ( - select key, value from src - UNION DISTINCT - select key, value from - ( - select key, value, count(1) from src group by key, value - UNION DISTINCT - select key, value, count(1) from src group by key, value - ) subq -) a - -UNION DISTINCT - -select key, value from src -) aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union30 -POSTHOOK: Lineage: union_subq_union30.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: union_subq_union30.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from union_subq_union30 order by key, value limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@union_subq_union30 -#### A masked pattern was here #### -POSTHOOK: query: select * from union_subq_union30 order by key, value limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@union_subq_union30 -#### A masked pattern was here #### -0 val_0 -2 val_2 -4 val_4 -5 val_5 -8 val_8 -9 val_9 -10 val_10 -11 val_11 -12 val_12 -15 val_15 -17 val_17 -18 val_18 -19 val_19 -20 val_20 -24 val_24 -26 val_26 -27 val_27 -28 val_28 -30 val_30 -33 val_33 -PREHOOK: query: drop table t1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table t1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table t2 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table t2 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table t1 as select * from src where key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t1 -POSTHOOK: query: create table t1 as select * from src where key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1 -POSTHOOK: Lineage: t1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table t2 as select * from src where key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t2 -POSTHOOK: query: create table t2 as select * from src where key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2 -POSTHOOK: Lineage: t2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table t3(key string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t3 -POSTHOOK: query: create table t3(key string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t3 -PREHOOK: query: create table t4(value string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t4 -POSTHOOK: query: create table t4(value string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t4 -PREHOOK: query: explain -from -(select * from t1 - UNION DISTINCT - select * from t2 -) x -insert overwrite table t3 - select key, count(1) group by key -insert overwrite table t4 - select value, count(1) group by value + Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 154 Data size: 27838 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Reducer 7 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 77 Data size: 13919 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 77 Data size: 13919 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s PREHOOK: type: QUERY -POSTHOOK: query: explain -from -(select * from t1 - UNION DISTINCT - select * from t2 -) x -insert overwrite table t3 - select key, count(1) group by key -insert overwrite table t4 - select value, count(1) group by value +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 +#### A masked pattern was here #### +0 1 +0 3 +2 1 +4 1 +5 1 +5 3 +8 1 +9 1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-2 + Stage: Stage-1 Tez #### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE), Union 4 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 + Truncated Path -> Alias: + /src2 [src2] + Map 6 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint #### A masked pattern was here #### - Vertices: - Map 1 + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src3 [src3] + Map 7 Map Operator Tree: TableScan - alias: t1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ + key expressions: _col0 (type: string) + null sort order: a + sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + tag: 0 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [a] + Map 9 Map Operator Tree: TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash + alias: b + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ + key expressions: _col0 (type: string) + null sort order: a + sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [b] Reducer 3 - Execution mode: llap + Needs Tagging: false Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t3 - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 4 - Execution mode: llap + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Reducer 5 + Needs Tagging: false Reduce Operator Tree: Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 108 Data size: 19391 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 108 Data size: 19391 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Reducer 8 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col2 + Position of Big Table: 1 + Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t4 + Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 216 Data size: 38782 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true Union 2 Vertex: Union 2 - - Stage: Stage-3 - Dependency Collection + Union 4 + Vertex: Union 4 Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t3 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t4 - - Stage: Stage-5 - Stats Work - Basic Stats Work: + Fetch Operator + limit: -1 + Processor Tree: + ListSink -PREHOOK: query: from -(select * from t1 - UNION DISTINCT - select * from t2 -) x -insert overwrite table t3 - select key, count(1) group by key -insert overwrite table t4 - select value, count(1) group by value -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t3 -PREHOOK: Output: default@t4 -POSTHOOK: query: from -(select * from t1 - UNION DISTINCT - select * from t2 -) x -insert overwrite table t3 - select key, count(1) group by key -insert overwrite table t4 - select value, count(1) group by value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t3 -POSTHOOK: Output: default@t4 -POSTHOOK: Lineage: t3.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] -POSTHOOK: Lineage: t3.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t4.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] -POSTHOOK: Lineage: t4.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: select * from t3 +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s PREHOOK: type: QUERY -PREHOOK: Input: default@t3 +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 #### A masked pattern was here #### -POSTHOOK: query: select * from t3 +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s POSTHOOK: type: QUERY -POSTHOOK: Input: default@t3 +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 #### A masked pattern was here #### -0 1 +0 3 2 1 4 1 -5 1 +5 3 8 1 9 1 -PREHOOK: query: select * from t4 -PREHOOK: type: QUERY -PREHOOK: Input: default@t4 -#### A masked pattern was here #### -POSTHOOK: query: select * from t4 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t4 -#### A masked pattern was here #### -val_5 1 -val_0 1 -val_2 1 -val_4 1 -val_8 1 -val_9 1 -PREHOOK: query: create table t5(c1 string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t5 -POSTHOOK: query: create table t5(c1 string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t5 -PREHOOK: query: create table t6(c1 string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t6 -POSTHOOK: query: create table t6(c1 string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t6 -PREHOOK: query: explain -from -( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, count(1) as cnt from t2 group by key -) x -insert overwrite table t5 - select c1, sum(cnt) group by c1 -insert overwrite table t6 - select c1, sum(cnt) group by c1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s PREHOOK: type: QUERY -POSTHOOK: query: explain -from -( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, count(1) as cnt from t2 group by key -) x -insert overwrite table t5 - select c1, sum(cnt) group by c1 -insert overwrite table t6 - select c1, sum(cnt) group by c1 +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-2 + Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Map 1 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: t1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash + alias: src2 + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 + Truncated Path -> Alias: + /src2 [src2] + Map 10 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) + null sort order: a sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [b] Map 6 Map Operator Tree: TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash + alias: src3 + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 37760 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src3 [src3] + Map 7 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 309 Data size: 54280 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) + null sort order: a sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap + Statistics: Num rows: 103 Data size: 18093 Basic stats: COMPLETE Column stats: NONE + tag: 0 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [a] + Reducer 3 + Needs Tagging: false Reduce Operator Tree: Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 18880 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Reducer 4 - Execution mode: llap + Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Reducer 5 + Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 5 - Execution mode: llap + Statistics: Num rows: 79 Data size: 14280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 79 Data size: 14280 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Reducer 8 + Needs Tagging: false Reduce Operator Tree: - Forward - Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t5 + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Position of Big Table: 0 + Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: string) - mode: complete + aggregations: count() + keys: _col0 (type: string) + mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t6 - Reducer 7 - Execution mode: llap + Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 113 Data size: 19902 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Reducer 9 + Needs Tagging: false Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Union 3 - Vertex: Union 3 - - Stage: Stage-3 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t5 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t6 - - Stage: Stage-5 - Stats Work - Basic Stats Work: - -PREHOOK: query: from -( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, count(1) as cnt from t2 group by key -) x -insert overwrite table t5 - select c1, sum(cnt) group by c1 -insert overwrite table t6 - select c1, sum(cnt) group by c1 -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t5 -PREHOOK: Output: default@t6 -POSTHOOK: query: from -( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, count(1) as cnt from t2 group by key -) x -insert overwrite table t5 - select c1, sum(cnt) group by c1 -insert overwrite table t6 - select c1, sum(cnt) group by c1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t5 -POSTHOOK: Output: default@t6 -POSTHOOK: Lineage: t5.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t5.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] -POSTHOOK: Lineage: t6.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t6.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] -PREHOOK: query: select * from t5 + Statistics: Num rows: 56 Data size: 9862 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 159 Data size: 28742 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s PREHOOK: type: QUERY -PREHOOK: Input: default@t5 +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 #### A masked pattern was here #### -POSTHOOK: query: select * from t5 +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s POSTHOOK: type: QUERY -POSTHOOK: Input: default@t5 +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 #### A masked pattern was here #### -8 1 -9 1 -4 1 -2 1 +0 1 0 3 +2 1 +4 1 +5 1 5 3 -PREHOOK: query: select * from t6 -PREHOOK: type: QUERY -PREHOOK: Input: default@t6 -#### A masked pattern was here #### -POSTHOOK: query: select * from t6 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t6 -#### A masked pattern was here #### 8 1 9 1 -4 1 -2 1 -0 3 -5 3 -PREHOOK: query: drop table t1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t1 -PREHOOK: Output: default@t1 -POSTHOOK: query: drop table t1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t1 -POSTHOOK: Output: default@t1 -PREHOOK: query: drop table t2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t2 -POSTHOOK: query: drop table t2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t2 -PREHOOK: query: create table t1 as select * from src where key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t1 -POSTHOOK: query: create table t1 as select * from src where key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1 -POSTHOOK: Lineage: t1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t2 -POSTHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2 -POSTHOOK: Lineage: t2.cnt EXPRESSION [(src)src.null, ] -POSTHOOK: Lineage: t2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: create table t7(c1 string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t7 -POSTHOOK: query: create table t7(c1 string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t7 -PREHOOK: query: create table t8(c1 string, cnt int) +PREHOOK: query: create table tmp_srcpart like srcpart PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@t8 -POSTHOOK: query: create table t8(c1 string, cnt int) +PREHOOK: Output: default@tmp_srcpart +POSTHOOK: query: create table tmp_srcpart like srcpart POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@t8 +POSTHOOK: Output: default@tmp_srcpart +PREHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: explain -from +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM ( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, cnt from t2 -) x -insert overwrite table t7 - select c1, count(1) group by c1 -insert overwrite table t8 - select c1, count(1) group by c1 -PREHOOK: type: QUERY + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION DISTINCT + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION DISTINCT + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain -from +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM ( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, cnt from t2 -) x -insert overwrite table t7 - select c1, count(1) group by c1 -insert overwrite table t8 - select c1, count(1) group by c1 -POSTHOOK: type: QUERY + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION DISTINCT + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION DISTINCT + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +POSTHOOK: type: CREATETABLE_AS_SELECT STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-2 + Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 5 (CONTAINS) + Map 7 <- Union 5 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: t1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: count() - keys: key (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE + Map 7 Map Operator Tree: TableScan - alias: t2 - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE + alias: b + Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), cnt (type: bigint) + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 360960 Basic stats: COMPLETE Column stats: NONE + Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Forward - Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: KEY._col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t7 + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator - aggregations: count(1) - keys: KEY._col0 (type: string) + aggregations: count() + keys: _col0 (type: string), _col1 (type: string) mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 375 Data size: 69750 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 375 Data size: 69750 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 375 Data size: 69750 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t8 - Union 3 - Vertex: Union 3 + name: default.tmp_unionall + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 180480 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 90240 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 90240 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 - Stage: Stage-3 + Stage: Stage-2 Dependency Collection - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t7 - Stage: Stage-4 - Stats Work - Basic Stats Work: - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t8 + Create Table Operator: + Create Table + columns: counts bigint, key string, value string + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmp_unionall - Stage: Stage-5 + Stage: Stage-3 Stats Work Basic Stats Work: -PREHOOK: query: from -( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, cnt from t2 -) x -insert overwrite table t7 - select c1, count(1) group by c1 -insert overwrite table t8 - select c1, count(1) group by c1 -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t7 -PREHOOK: Output: default@t8 -POSTHOOK: query: from -( - select key as c1, count(1) as cnt from t1 group by key - UNION DISTINCT - select key as c1, cnt from t2 -) x -insert overwrite table t7 - select c1, count(1) group by c1 -insert overwrite table t8 - select c1, count(1) group by c1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t7 -POSTHOOK: Output: default@t8 -POSTHOOK: Lineage: t7.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t7.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] -POSTHOOK: Lineage: t8.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t8.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] -PREHOOK: query: select * from t7 -PREHOOK: type: QUERY -PREHOOK: Input: default@t7 -#### A masked pattern was here #### -POSTHOOK: query: select * from t7 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t7 -#### A masked pattern was here #### -8 1 -9 1 -4 1 -2 1 -0 1 -5 1 -PREHOOK: query: select * from t8 -PREHOOK: type: QUERY -PREHOOK: Input: default@t8 -#### A masked pattern was here #### -POSTHOOK: query: select * from t8 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t8 + Stage: Stage-0 + Move Operator + files: + hdfs directory: true #### A masked pattern was here #### -8 1 -9 1 -4 1 -2 1 -0 1 -5 1 -PREHOOK: query: drop table if exists t1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t1 -PREHOOK: Output: default@t1 -POSTHOOK: query: drop table if exists t1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t1 -POSTHOOK: Output: default@t1 -PREHOOK: query: drop table if exists t2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t2 -POSTHOOK: query: drop table if exists t2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t2 -PREHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t1 -POSTHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1 -POSTHOOK: Lineage: t1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t2 -POSTHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2 -POSTHOOK: Lineage: t2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] + PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1 +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + UNION DISTINCT -SELECT CAST(key AS BIGINT) AS key FROM t2) a + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1 +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + UNION DISTINCT -SELECT CAST(key AS BIGINT) AS key FROM t2) a + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -12933,370 +6376,1367 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) + Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: t1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(key) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 4 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(UDFToLong(key)) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Lateral View Forward + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: array(1,2,3) (type: array) outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE Reducer 3 - Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: double) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + outputColumnNames: _col0, _col1 + Statistics: Num rows: 775 Data size: 8233 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1 0 val_0 +1 10 val_10 +1 100 val_100 +1 103 val_103 +1 104 val_104 +1 105 val_105 +1 11 val_11 +1 111 val_111 +1 113 val_113 +1 114 val_114 +1 116 val_116 +1 118 val_118 +1 119 val_119 +1 12 val_12 +1 120 val_120 +1 125 val_125 +1 126 val_126 +1 128 val_128 +1 129 val_129 +1 131 val_131 +1 133 val_133 +1 134 val_134 +1 136 val_136 +1 137 val_137 +1 138 val_138 +1 143 val_143 +1 145 val_145 +1 146 val_146 +1 149 val_149 +1 15 val_15 +1 150 val_150 +1 152 val_152 +1 153 val_153 +1 155 val_155 +1 156 val_156 +1 157 val_157 +1 158 val_158 +1 160 val_160 +1 162 val_162 +1 163 val_163 +1 164 val_164 +1 165 val_165 +1 166 val_166 +1 167 val_167 +1 168 val_168 +1 169 val_169 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 174 val_174 +1 175 val_175 +1 176 val_176 +1 177 val_177 +1 178 val_178 +1 179 val_179 +1 18 val_18 +1 180 val_180 +1 181 val_181 +1 183 val_183 +1 186 val_186 +1 187 val_187 +1 189 val_189 +1 19 val_19 +1 190 val_190 +1 191 val_191 +1 192 val_192 +1 193 val_193 +1 194 val_194 +1 195 val_195 +1 196 val_196 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 20 val_20 +1 200 val_200 +1 201 val_201 +1 202 val_202 +1 203 val_203 +1 205 val_205 +1 207 val_207 +1 208 val_208 +1 209 val_209 +1 213 val_213 +1 214 val_214 +1 216 val_216 +1 217 val_217 +1 218 val_218 +1 219 val_219 +1 221 val_221 +1 222 val_222 +1 223 val_223 +1 224 val_224 +1 226 val_226 +1 228 val_228 +1 229 val_229 +1 230 val_230 +1 233 val_233 +1 235 val_235 +1 237 val_237 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 241 val_241 +1 242 val_242 +1 244 val_244 +1 247 val_247 +1 248 val_248 +1 249 val_249 +1 252 val_252 +1 255 val_255 +1 256 val_256 +1 257 val_257 +1 258 val_258 +1 26 val_26 +1 260 val_260 +1 262 val_262 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 27 val_27 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 275 val_275 +1 277 val_277 +1 278 val_278 +1 28 val_28 +1 280 val_280 +1 281 val_281 +1 282 val_282 +1 283 val_283 +1 284 val_284 +1 285 val_285 +1 286 val_286 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 291 val_291 +1 292 val_292 +1 296 val_296 +1 298 val_298 +1 30 val_30 +1 302 val_302 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 309 val_309 +1 310 val_310 +1 311 val_311 +1 315 val_315 +1 316 val_316 +1 317 val_317 +1 318 val_318 +1 321 val_321 +1 322 val_322 +1 323 val_323 +1 325 val_325 +1 327 val_327 +1 33 val_33 +1 331 val_331 +1 332 val_332 +1 333 val_333 +1 335 val_335 +1 336 val_336 +1 338 val_338 +1 339 val_339 +1 34 val_34 +1 341 val_341 +1 342 val_342 +1 344 val_344 +1 345 val_345 +1 348 val_348 +1 35 val_35 +1 351 val_351 +1 353 val_353 +1 356 val_356 +1 360 val_360 +1 362 val_362 +1 364 val_364 +1 365 val_365 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 369 val_369 +1 37 val_37 +1 373 val_373 +1 374 val_374 +1 375 val_375 +1 377 val_377 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 384 val_384 +1 386 val_386 +1 389 val_389 +1 392 val_392 +1 393 val_393 +1 394 val_394 +1 395 val_395 +1 396 val_396 +1 397 val_397 +1 399 val_399 +1 4 val_4 +1 400 val_400 +1 401 val_401 +1 402 val_402 +1 403 val_403 +1 404 val_404 +1 406 val_406 +1 407 val_407 +1 409 val_409 +1 41 val_41 +1 411 val_411 +1 413 val_413 +1 414 val_414 +1 417 val_417 +1 418 val_418 +1 419 val_419 +1 42 val_42 +1 421 val_421 +1 424 val_424 +1 427 val_427 +1 429 val_429 +1 43 val_43 +1 430 val_430 +1 431 val_431 +1 432 val_432 +1 435 val_435 +1 436 val_436 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 44 val_44 +1 443 val_443 +1 444 val_444 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 452 val_452 +1 453 val_453 +1 454 val_454 +1 455 val_455 +1 457 val_457 +1 458 val_458 +1 459 val_459 +1 460 val_460 +1 462 val_462 +1 463 val_463 +1 466 val_466 +1 467 val_467 +1 468 val_468 +1 469 val_469 +1 47 val_47 +1 470 val_470 +1 472 val_472 +1 475 val_475 +1 477 val_477 +1 478 val_478 +1 479 val_479 +1 480 val_480 +1 481 val_481 +1 482 val_482 +1 483 val_483 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 489 val_489 +1 490 val_490 +1 491 val_491 +1 492 val_492 +1 493 val_493 +1 494 val_494 +1 495 val_495 +1 496 val_496 +1 497 val_497 +1 498 val_498 +1 5 val_5 +1 51 val_51 +1 53 val_53 +1 54 val_54 +1 57 val_57 +1 58 val_58 +1 64 val_64 +1 65 val_65 +1 66 val_66 +1 67 val_67 +1 69 val_69 +1 70 val_70 +1 72 val_72 +1 74 val_74 +1 76 val_76 +1 77 val_77 +1 78 val_78 +1 8 val_8 +1 80 val_80 +1 82 val_82 +1 83 val_83 +1 84 val_84 +1 85 val_85 +1 86 val_86 +1 87 val_87 +1 9 val_9 +1 90 val_90 +1 92 val_92 +1 95 val_95 +1 96 val_96 +1 97 val_97 +1 98 val_98 +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key -PREHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1 UNION DISTINCT -SELECT CAST(key AS BIGINT) AS key FROM t2) a + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1 -UNION DISTINCT -SELECT CAST(key AS BIGINT) AS key FROM t2) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### -8.0 -0.0 -9.0 -2.0 -4.0 -5.0 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key -UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key FROM t2) a -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key FROM t2) a -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Map 4 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(UDFToLong(_col0)) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(key) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 +1 0 val_0 +1 103 val_103 +1 105 val_105 +1 116 val_116 +1 120 val_120 +1 129 val_129 +1 131 val_131 +1 136 val_136 +1 155 val_155 +1 157 val_157 +1 160 val_160 +1 180 val_180 +1 183 val_183 +1 190 val_190 +1 191 val_191 +1 193 val_193 +1 205 val_205 +1 207 val_207 +1 213 val_213 +1 223 val_223 +1 235 val_235 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 248 val_248 +1 260 val_260 +1 28 val_28 +1 286 val_286 +1 298 val_298 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 315 val_315 +1 317 val_317 +1 321 val_321 +1 33 val_33 +1 331 val_331 +1 356 val_356 +1 360 val_360 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 37 val_37 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 389 val_389 +1 395 val_395 +1 400 val_400 +1 402 val_402 +1 406 val_406 +1 407 val_407 +1 42 val_42 +1 430 val_430 +1 436 val_436 +1 44 val_44 +1 452 val_452 +1 466 val_466 +1 467 val_467 +1 472 val_472 +1 480 val_480 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 495 val_495 +1 496 val_496 +1 498 val_498 +1 51 val_51 +1 57 val_57 +1 58 val_58 +1 65 val_65 +1 66 val_66 +1 69 val_69 +1 70 val_70 +1 78 val_78 +1 8 val_8 +1 92 val_92 +1 96 val_96 +1 104 val_104 +1 111 val_111 +1 114 val_114 +1 118 val_118 +1 12 val_12 +1 133 val_133 +1 138 val_138 +1 153 val_153 +1 156 val_156 +1 162 val_162 +1 168 val_168 +1 174 val_174 +1 176 val_176 +1 178 val_178 +1 181 val_181 +1 186 val_186 +1 187 val_187 +1 192 val_192 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 201 val_201 +1 203 val_203 +1 208 val_208 +1 214 val_214 +1 218 val_218 +1 219 val_219 +1 230 val_230 +1 237 val_237 +1 249 val_249 +1 256 val_256 +1 257 val_257 +1 277 val_277 +1 278 val_278 +1 284 val_284 +1 285 val_285 +1 310 val_310 +1 322 val_322 +1 327 val_327 +1 333 val_333 +1 336 val_336 +1 338 val_338 +1 341 val_341 +1 344 val_344 +1 348 val_348 +1 351 val_351 +1 362 val_362 +1 364 val_364 +1 374 val_374 +1 392 val_392 +1 397 val_397 +1 4 val_4 +1 403 val_403 +1 404 val_404 +1 419 val_419 +1 421 val_421 +1 427 val_427 +1 429 val_429 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 454 val_454 +1 455 val_455 +1 460 val_460 +1 463 val_463 +1 470 val_470 +1 478 val_478 +1 479 val_479 +1 489 val_489 +1 491 val_491 +1 492 val_492 +1 53 val_53 +1 67 val_67 +1 82 val_82 +1 90 val_90 +1 97 val_97 +1 10 val_10 +1 125 val_125 +1 134 val_134 +1 143 val_143 +1 145 val_145 +1 149 val_149 +1 150 val_150 +1 158 val_158 +1 163 val_163 +1 164 val_164 +1 166 val_166 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 189 val_189 +1 19 val_19 +1 195 val_195 +1 196 val_196 +1 20 val_20 +1 209 val_209 +1 216 val_216 +1 217 val_217 +1 224 val_224 +1 228 val_228 +1 229 val_229 +1 233 val_233 +1 241 val_241 +1 244 val_244 +1 247 val_247 +1 255 val_255 +1 258 val_258 +1 26 val_26 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 281 val_281 +1 291 val_291 +1 296 val_296 +1 30 val_30 +1 302 val_302 +1 309 val_309 +1 316 val_316 +1 318 val_318 +1 325 val_325 +1 332 val_332 +1 335 val_335 +1 339 val_339 +1 342 val_342 +1 345 val_345 +1 353 val_353 +1 369 val_369 +1 373 val_373 +1 375 val_375 +1 377 val_377 +1 384 val_384 +1 386 val_386 +1 394 val_394 +1 396 val_396 +1 399 val_399 +1 401 val_401 +1 41 val_41 +1 413 val_413 +1 414 val_414 +1 431 val_431 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 459 val_459 +1 462 val_462 +1 468 val_468 +1 47 val_47 +1 482 val_482 +1 490 val_490 +1 493 val_493 +1 494 val_494 +1 497 val_497 +1 5 val_5 +1 54 val_54 +1 74 val_74 +1 77 val_77 +1 80 val_80 +1 84 val_84 +1 85 val_85 +1 87 val_87 +1 9 val_9 +1 95 val_95 +1 100 val_100 +1 11 val_11 +1 113 val_113 +1 119 val_119 +1 126 val_126 +1 128 val_128 +1 137 val_137 +1 146 val_146 +1 15 val_15 +1 152 val_152 +1 165 val_165 +1 167 val_167 +1 169 val_169 +1 175 val_175 +1 177 val_177 +1 179 val_179 +1 18 val_18 +1 194 val_194 +1 200 val_200 +1 202 val_202 +1 221 val_221 +1 222 val_222 +1 226 val_226 +1 242 val_242 +1 252 val_252 +1 262 val_262 +1 27 val_27 +1 275 val_275 +1 280 val_280 +1 282 val_282 +1 283 val_283 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 292 val_292 +1 311 val_311 +1 323 val_323 +1 34 val_34 +1 35 val_35 +1 365 val_365 +1 393 val_393 +1 409 val_409 +1 411 val_411 +1 417 val_417 +1 418 val_418 +1 424 val_424 +1 43 val_43 +1 432 val_432 +1 435 val_435 +1 443 val_443 +1 444 val_444 +1 453 val_453 +1 457 val_457 +1 458 val_458 +1 469 val_469 +1 475 val_475 +1 477 val_477 +1 481 val_481 +1 483 val_483 +1 64 val_64 +1 72 val_72 +1 76 val_76 +1 83 val_83 +1 86 val_86 +1 98 val_98 +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key -PREHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key FROM t2) a + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key FROM t2) a + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### -8.0 -0.0 -9.0 -2.0 -4.0 -5.0 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +1 0 val_0 +1 103 val_103 +1 105 val_105 +1 116 val_116 +1 120 val_120 +1 129 val_129 +1 131 val_131 +1 136 val_136 +1 155 val_155 +1 157 val_157 +1 160 val_160 +1 180 val_180 +1 183 val_183 +1 190 val_190 +1 191 val_191 +1 193 val_193 +1 205 val_205 +1 207 val_207 +1 213 val_213 +1 223 val_223 +1 235 val_235 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 248 val_248 +1 260 val_260 +1 28 val_28 +1 286 val_286 +1 298 val_298 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 315 val_315 +1 317 val_317 +1 321 val_321 +1 33 val_33 +1 331 val_331 +1 356 val_356 +1 360 val_360 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 37 val_37 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 389 val_389 +1 395 val_395 +1 400 val_400 +1 402 val_402 +1 406 val_406 +1 407 val_407 +1 42 val_42 +1 430 val_430 +1 436 val_436 +1 44 val_44 +1 452 val_452 +1 466 val_466 +1 467 val_467 +1 472 val_472 +1 480 val_480 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 495 val_495 +1 496 val_496 +1 498 val_498 +1 51 val_51 +1 57 val_57 +1 58 val_58 +1 65 val_65 +1 66 val_66 +1 69 val_69 +1 70 val_70 +1 78 val_78 +1 8 val_8 +1 92 val_92 +1 96 val_96 +1 104 val_104 +1 111 val_111 +1 114 val_114 +1 118 val_118 +1 12 val_12 +1 133 val_133 +1 138 val_138 +1 153 val_153 +1 156 val_156 +1 162 val_162 +1 168 val_168 +1 174 val_174 +1 176 val_176 +1 178 val_178 +1 181 val_181 +1 186 val_186 +1 187 val_187 +1 192 val_192 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 201 val_201 +1 203 val_203 +1 208 val_208 +1 214 val_214 +1 218 val_218 +1 219 val_219 +1 230 val_230 +1 237 val_237 +1 249 val_249 +1 256 val_256 +1 257 val_257 +1 277 val_277 +1 278 val_278 +1 284 val_284 +1 285 val_285 +1 310 val_310 +1 322 val_322 +1 327 val_327 +1 333 val_333 +1 336 val_336 +1 338 val_338 +1 341 val_341 +1 344 val_344 +1 348 val_348 +1 351 val_351 +1 362 val_362 +1 364 val_364 +1 374 val_374 +1 392 val_392 +1 397 val_397 +1 4 val_4 +1 403 val_403 +1 404 val_404 +1 419 val_419 +1 421 val_421 +1 427 val_427 +1 429 val_429 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 454 val_454 +1 455 val_455 +1 460 val_460 +1 463 val_463 +1 470 val_470 +1 478 val_478 +1 479 val_479 +1 489 val_489 +1 491 val_491 +1 492 val_492 +1 53 val_53 +1 67 val_67 +1 82 val_82 +1 90 val_90 +1 97 val_97 +1 10 val_10 +1 125 val_125 +1 134 val_134 +1 143 val_143 +1 145 val_145 +1 149 val_149 +1 150 val_150 +1 158 val_158 +1 163 val_163 +1 164 val_164 +1 166 val_166 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 189 val_189 +1 19 val_19 +1 195 val_195 +1 196 val_196 +1 20 val_20 +1 209 val_209 +1 216 val_216 +1 217 val_217 +1 224 val_224 +1 228 val_228 +1 229 val_229 +1 233 val_233 +1 241 val_241 +1 244 val_244 +1 247 val_247 +1 255 val_255 +1 258 val_258 +1 26 val_26 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 281 val_281 +1 291 val_291 +1 296 val_296 +1 30 val_30 +1 302 val_302 +1 309 val_309 +1 316 val_316 +1 318 val_318 +1 325 val_325 +1 332 val_332 +1 335 val_335 +1 339 val_339 +1 342 val_342 +1 345 val_345 +1 353 val_353 +1 369 val_369 +1 373 val_373 +1 375 val_375 +1 377 val_377 +1 384 val_384 +1 386 val_386 +1 394 val_394 +1 396 val_396 +1 399 val_399 +1 401 val_401 +1 41 val_41 +1 413 val_413 +1 414 val_414 +1 431 val_431 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 459 val_459 +1 462 val_462 +1 468 val_468 +1 47 val_47 +1 482 val_482 +1 490 val_490 +1 493 val_493 +1 494 val_494 +1 497 val_497 +1 5 val_5 +1 54 val_54 +1 74 val_74 +1 77 val_77 +1 80 val_80 +1 84 val_84 +1 85 val_85 +1 87 val_87 +1 9 val_9 +1 95 val_95 +1 100 val_100 +1 11 val_11 +1 113 val_113 +1 119 val_119 +1 126 val_126 +1 128 val_128 +1 137 val_137 +1 146 val_146 +1 15 val_15 +1 152 val_152 +1 165 val_165 +1 167 val_167 +1 169 val_169 +1 175 val_175 +1 177 val_177 +1 179 val_179 +1 18 val_18 +1 194 val_194 +1 200 val_200 +1 202 val_202 +1 221 val_221 +1 222 val_222 +1 226 val_226 +1 242 val_242 +1 252 val_252 +1 262 val_262 +1 27 val_27 +1 275 val_275 +1 280 val_280 +1 282 val_282 +1 283 val_283 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 292 val_292 +1 311 val_311 +1 323 val_323 +1 34 val_34 +1 35 val_35 +1 365 val_365 +1 393 val_393 +1 409 val_409 +1 411 val_411 +1 417 val_417 +1 418 val_418 +1 424 val_424 +1 43 val_43 +1 432 val_432 +1 435 val_435 +1 443 val_443 +1 444 val_444 +1 453 val_453 +1 457 val_457 +1 458 val_458 +1 469 val_469 +1 475 val_475 +1 477 val_477 +1 481 val_481 +1 483 val_483 +1 64 val_64 +1 72 val_72 +1 76 val_76 +1 83 val_83 +1 86 val_86 +1 98 val_98 +PREHOOK: query: create table jackson_sev_same as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_same +POSTHOOK: query: create table jackson_sev_same as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_same +POSTHOOK: Lineage: jackson_sev_same.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: jackson_sev_same.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table dim_pho as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@dim_pho +POSTHOOK: query: create table dim_pho as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dim_pho +POSTHOOK: Lineage: dim_pho.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dim_pho.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table jackson_sev_add as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_add +POSTHOOK: query: create table jackson_sev_add as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_add +POSTHOOK: Lineage: jackson_sev_add.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: jackson_sev_add.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(key) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Map 5 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(UDFToLong(_col0)) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 3 <- Union 4 (CONTAINS) + Map 6 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Union 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) = 97.0) (type: boolean) + Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap + Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: dim_pho + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 97.0) (type: boolean) + Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: jackson_sev_add + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 97.0) (type: boolean) + Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 275 Data size: 48171 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 48171 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 48171 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: double) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 87584 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Union 4 + Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -13304,193 +7744,330 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 +PREHOOK: Input: default@dim_pho +PREHOOK: Input: default@jackson_sev_add +PREHOOK: Input: default@jackson_sev_same #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@dim_pho +POSTHOOK: Input: default@jackson_sev_add +POSTHOOK: Input: default@jackson_sev_same #### A masked pattern was here #### -8.0 -0.0 -9.0 -2.0 -4.0 -5.0 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS STRING) AS value FROM t1 a JOIN t2 b ON a.key = b.key -UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +97 val_97 +97 val_97 +PREHOOK: query: create table union_subq_union(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: create table union_subq_union(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union +PREHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS STRING) AS value FROM t1 a JOIN t2 b ON a.key = b.key -UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +POSTHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) + Map 1 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) + Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 6 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: a - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - input vertices: - 1 Map 4 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Map 8 Map Operator Tree: TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToDouble(key) (type: double), key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col0 (type: double), _col1 (type: string) + aggregations: count() + keys: key (type: string), value (type: string) mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: double), KEY._col1 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE Union 2 Vertex: Union 2 + Union 6 + Vertex: Union 6 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union -PREHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key -UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union +POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key -UNION DISTINCT -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +POSTHOOK: query: select * from union_subq_union order by key, value limit 20 POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@union_subq_union #### A masked pattern was here #### -9.0 9 -0.0 0 -2.0 2 -5.0 5 -8.0 8 -4.0 4 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: create table union_subq_union29(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union29 +POSTHOOK: query: create table union_subq_union29(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union29 +PREHOOK: query: explain +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: query: explain +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -13498,336 +8075,517 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Map 1 <- Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 4 <- Union 5 (CONTAINS) + Map 7 <- Union 5 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToDouble(key) (type: double), key (type: string) + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col0 (type: double), _col1 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Map 4 Map Operator Tree: TableScan - alias: a - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - input vertices: - 1 Map 5 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToString(CAST( _col1 AS varchar(20))) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map 7 Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: double), KEY._col1 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union29 + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Union 2 Vertex: Union 2 + Union 5 + Vertex: Union 5 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union29 -PREHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union29 +POSTHOOK: query: insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union29 +POSTHOOK: Lineage: union_subq_union29.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union29.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union29 order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union29 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 -UNION DISTINCT -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: query: select * from union_subq_union29 order by key, value limit 20 POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@union_subq_union29 #### A masked pattern was here #### -9.0 9 -0.0 0 -2.0 2 -5.0 5 -8.0 8 -4.0 4 -PREHOOK: query: drop table if exists test_src -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists test_src -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE test_src (key STRING, value STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_src -POSTHOOK: query: CREATE TABLE test_src (key STRING, value STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_src -PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION DISTINCT - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: explain +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION DISTINCT - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a +POSTHOOK: query: explain +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 10 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 8 <- Union 7 (SIMPLE_EDGE) + Reducer 9 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 0.0) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Map 11 Map Operator Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 10 + Reduce Operator Tree: + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 4 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 12 + Reduce Operator Tree: + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 2 + Reduce Operator Tree: + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src - Reducer 5 - Execution mode: llap + Reducer 9 Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), UDFToString(_col1) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 3 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + keys: _col0 (type: int) mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - - Stage: Stage-2 - Dependency Collection + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Union 7 + Vertex: Union 7 Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src - - Stage: Stage-3 - Stats Work - Basic Stats Work: + Fetch Operator + limit: -1 + Processor Tree: + ListSink -PREHOOK: query: INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION DISTINCT - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a +PREHOOK: query: CREATE TABLE union_out (id int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_out +POSTHOOK: query: CREATE TABLE union_out (id int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_out +PREHOOK: query: insert overwrite table union_out +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: default@test_src -POSTHOOK: query: INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION DISTINCT - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a +PREHOOK: Output: default@union_out +POSTHOOK: query: insert overwrite table union_out +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_src -POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.null, ] -PREHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: Output: default@union_out +POSTHOOK: Lineage: union_out.id EXPRESSION [] +PREHOOK: query: select * from union_out PREHOOK: type: QUERY -PREHOOK: Input: default@test_src +PREHOOK: Input: default@union_out #### A masked pattern was here #### -POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: query: select * from union_out POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_src +POSTHOOK: Input: default@union_out #### A masked pattern was here #### -310 -PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key +4 +1 +2 +3 +PREHOOK: query: create table union_subq_union30(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union30 +POSTHOOK: query: create table union_subq_union30(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union30 +PREHOOK: query: explain +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + UNION DISTINCT - SELECT key, value FROM src - WHERE key = 0 -)a + +select key, value from src +) aa PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key +POSTHOOK: query: explain +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + UNION DISTINCT - SELECT key, value FROM src - WHERE key = 0 -)a + +select key, value from src +) aa POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -13840,99 +8598,197 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Map 1 <- Union 2 (CONTAINS) + Map 12 <- Union 4 (CONTAINS) + Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 2 (CONTAINS), Union 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map 10 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - keys: key (type: string) + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Map 12 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 11 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 0.0) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 Execution mode: llap Reduce Operator Tree: Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), UDFToString(_col1) (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union30 + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src - Union 3 - Vertex: Union 3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 8 + Vertex: Union 8 Stage: Stage-2 Dependency Collection @@ -13945,230 +8801,177 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src + name: default.union_subq_union30 Stage: Stage-3 Stats Work Basic Stats Work: -PREHOOK: query: INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key +PREHOOK: query: insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + UNION DISTINCT - SELECT key, value FROM src - WHERE key = 0 -)a + +select key, value from src +) aa PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: default@test_src -POSTHOOK: query: INSERT OVERWRITE TABLE test_src -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key +PREHOOK: Output: default@union_subq_union30 +POSTHOOK: query: insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + UNION DISTINCT - SELECT key, value FROM src - WHERE key = 0 -)a + +select key, value from src +) aa POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_src -POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.null, (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: Output: default@union_subq_union30 +POSTHOOK: Lineage: union_subq_union30.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union30.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union30 order by key, value limit 20 PREHOOK: type: QUERY -PREHOOK: Input: default@test_src +PREHOOK: Input: default@union_subq_union30 #### A masked pattern was here #### -POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: query: select * from union_subq_union30 order by key, value limit 20 POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_src +POSTHOOK: Input: default@union_subq_union30 #### A masked pattern was here #### -310 -PREHOOK: query: -- union34.q - -create table src10_1 (key string, value string) -PREHOOK: type: CREATETABLE +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table t2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table t2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src PREHOOK: Output: database:default -PREHOOK: Output: default@src10_1 -POSTHOOK: query: -- union34.q - -create table src10_1 (key string, value string) -POSTHOOK: type: CREATETABLE +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_1 -PREHOOK: query: create table src10_2 (key string, value string) -PREHOOK: type: CREATETABLE +POSTHOOK: Output: default@t1 +POSTHOOK: Lineage: t1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table t2 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src PREHOOK: Output: database:default -PREHOOK: Output: default@src10_2 -POSTHOOK: query: create table src10_2 (key string, value string) -POSTHOOK: type: CREATETABLE +PREHOOK: Output: default@t2 +POSTHOOK: query: create table t2 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_2 -PREHOOK: query: create table src10_3 (key string, value string) +POSTHOOK: Output: default@t2 +POSTHOOK: Lineage: t2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table t3(key string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@src10_3 -POSTHOOK: query: create table src10_3 (key string, value string) +PREHOOK: Output: default@t3 +POSTHOOK: query: create table t3(key string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_3 -PREHOOK: query: create table src10_4 (key string, value string) +POSTHOOK: Output: default@t3 +PREHOOK: query: create table t4(value string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@src10_4 -POSTHOOK: query: create table src10_4 (key string, value string) +PREHOOK: Output: default@t4 +POSTHOOK: query: create table t4(value string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_4 -PREHOOK: query: from (select * from src tablesample (10 rows)) a -insert overwrite table src10_1 select * -insert overwrite table src10_2 select * -insert overwrite table src10_3 select * -insert overwrite table src10_4 select * -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@src10_1 -PREHOOK: Output: default@src10_2 -PREHOOK: Output: default@src10_3 -PREHOOK: Output: default@src10_4 -POSTHOOK: query: from (select * from src tablesample (10 rows)) a -insert overwrite table src10_1 select * -insert overwrite table src10_2 select * -insert overwrite table src10_3 select * -insert overwrite table src10_4 select * -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@src10_1 -POSTHOOK: Output: default@src10_2 -POSTHOOK: Output: default@src10_3 -POSTHOOK: Output: default@src10_4 -POSTHOOK: Lineage: src10_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: src10_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: src10_3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: src10_4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: analyze table src10_1 compute statistics -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_1 -PREHOOK: Output: default@src10_1 -POSTHOOK: query: analyze table src10_1 compute statistics -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_1 -POSTHOOK: Output: default@src10_1 -PREHOOK: query: analyze table src10_2 compute statistics -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_2 -PREHOOK: Output: default@src10_2 -POSTHOOK: query: analyze table src10_2 compute statistics -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_2 -POSTHOOK: Output: default@src10_2 -PREHOOK: query: analyze table src10_3 compute statistics -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_3 -PREHOOK: Output: default@src10_3 -POSTHOOK: query: analyze table src10_3 compute statistics -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_3 -POSTHOOK: Output: default@src10_3 -PREHOOK: query: analyze table src10_4 compute statistics -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_4 -PREHOOK: Output: default@src10_4 -POSTHOOK: query: analyze table src10_4 compute statistics -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_4 -POSTHOOK: Output: default@src10_4 +POSTHOOK: Output: default@t4 PREHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value PREHOOK: type: QUERY POSTHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-2 Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 6 (CONTAINS) - Map 8 <- Union 6 (CONTAINS) + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src10_1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - input vertices: - 1 Map 4 - Statistics: Num rows: 11 Data size: 4048 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src10_2 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src10_3 + alias: t1 Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -14182,14 +8985,12 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 8 + Map 5 Map Operator Tree: TableScan - alias: src10_4 + alias: t2 Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -14203,442 +9004,642 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs Reducer 3 - Execution mode: llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3 + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 4 Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash + Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4 Union 2 Vertex: Union 2 - Union 6 - Vertex: Union 6 + + Stage: Stage-3 + Dependency Collection Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3 -PREHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 + Stage: Stage-4 + Stats Work + Basic Stats Work: + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + +PREHOOK: query: from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value PREHOOK: type: QUERY -PREHOOK: Input: default@src10_1 -PREHOOK: Input: default@src10_2 -PREHOOK: Input: default@src10_3 -PREHOOK: Input: default@src10_4 +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t3 +PREHOOK: Output: default@t4 +POSTHOOK: query: from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t3 +POSTHOOK: Output: default@t4 +POSTHOOK: Lineage: t3.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t3.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t4.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t4.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from t3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 +POSTHOOK: query: select * from t3 POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_1 -POSTHOOK: Input: default@src10_2 -POSTHOOK: Input: default@src10_3 -POSTHOOK: Input: default@src10_4 +POSTHOOK: Input: default@t3 #### A masked pattern was here #### -238 val_238 -484 val_484 -278 val_278 -255 val_255 -165 val_165 -27 val_27 -311 val_311 -409 val_409 -86 val_86 -98 val_98 +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: select * from t4 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +POSTHOOK: query: select * from t4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t4 +#### A masked pattern was here #### +val_0 1 +val_2 1 +val_4 1 +val_5 1 +val_8 1 +val_9 1 +PREHOOK: query: create table t5(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t5 +POSTHOOK: query: create table t5(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t5 +PREHOOK: query: create table t6(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t6 +POSTHOOK: query: create table t6(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t6 PREHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 PREHOOK: type: QUERY POSTHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-2 Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 7 (CONTAINS) - Map 9 <- Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 8 <- Union 3 (CONTAINS), Union 7 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src10_1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src10_2 + alias: t1 Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src10_3 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + aggregations: count() + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Map 6 Map Operator Tree: TableScan - alias: src10_4 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + alias: t2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + aggregations: count() + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reducer 2 - Execution mode: llap Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 4048 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reducer 4 - Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 - Execution mode: llap + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 5 + Reduce Operator Tree: + Forward + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5 + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6 + Reducer 7 Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Union 3 Vertex: Union 3 - Union 7 - Vertex: Union 7 + + Stage: Stage-3 + Dependency Collection Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5 -PREHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 + Stage: Stage-4 + Stats Work + Basic Stats Work: + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 PREHOOK: type: QUERY -PREHOOK: Input: default@src10_1 -PREHOOK: Input: default@src10_2 -PREHOOK: Input: default@src10_3 -PREHOOK: Input: default@src10_4 +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t5 +PREHOOK: Output: default@t6 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t5 +POSTHOOK: Output: default@t6 +POSTHOOK: Lineage: t5.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t5.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t6.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t6.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +PREHOOK: query: select * from t5 +PREHOOK: type: QUERY +PREHOOK: Input: default@t5 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) - UNION DISTINCT - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 -) alias1 +POSTHOOK: query: select * from t5 POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_1 -POSTHOOK: Input: default@src10_2 -POSTHOOK: Input: default@src10_3 -POSTHOOK: Input: default@src10_4 +POSTHOOK: Input: default@t5 #### A masked pattern was here #### -238 val_238 -484 val_484 -278 val_278 -255 val_255 -165 val_165 -27 val_27 -311 val_311 -409 val_409 -86 val_86 -98 val_98 -PREHOOK: query: drop table if exists tmptable -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@tmptable -PREHOOK: Output: default@tmptable -POSTHOOK: query: drop table if exists tmptable -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@tmptable -POSTHOOK: Output: default@tmptable -PREHOOK: query: create table tmptable(key string, value int) +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: select * from t6 +PREHOOK: type: QUERY +PREHOOK: Input: default@t6 +#### A masked pattern was here #### +POSTHOOK: query: select * from t6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t6 +#### A masked pattern was here #### +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: create table t9 as select key, count(1) as cnt from src where key < 10 group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t9 +POSTHOOK: query: create table t9 as select key, count(1) as cnt from src where key < 10 group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t9 +POSTHOOK: Lineage: t9.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: t9.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: create table t7(c1 string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@tmptable -POSTHOOK: query: create table tmptable(key string, value int) +PREHOOK: Output: default@t7 +POSTHOOK: query: create table t7(c1 string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@tmptable -PREHOOK: query: explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc +POSTHOOK: Output: default@t7 +PREHOOK: query: create table t8(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t8 +POSTHOOK: query: create table t8(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t8 +PREHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t9 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 PREHOOK: type: QUERY -POSTHOOK: query: explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc +POSTHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t9 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-2 Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Map 6 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: t1 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() + keys: key (type: string) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Map 6 Map Operator Tree: TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: t9 + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string), cnt (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count() + keys: _col0 (type: string), _col1 (type: bigint) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE Reducer 2 - Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 11 Data size: 2072 Basic stats: COMPLETE Column stats: NONE Reducer 4 - Execution mode: llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable - Reducer 6 - Execution mode: llap + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE + Reducer 5 Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst2' (type: string), _col0 (type: bigint) + Forward + Statistics: Num rows: 5 Data size: 941 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t7 + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8 Union 3 Vertex: Union 3 - Stage: Stage-2 + Stage: Stage-3 Dependency Collection Stage: Stage-0 @@ -14649,47 +9650,99 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable + name: default.t7 - Stage: Stage-3 + Stage: Stage-4 Stats Work Basic Stats Work: -PREHOOK: query: insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t9 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t9 +PREHOOK: Output: default@t7 +PREHOOK: Output: default@t8 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t9 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t9 +POSTHOOK: Output: default@t7 +POSTHOOK: Output: default@t8 +POSTHOOK: Lineage: t7.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t9)t9.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t7.cnt EXPRESSION [(t1)t1.null, (t9)t9.null, ] +POSTHOOK: Lineage: t8.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t9)t9.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t8.cnt EXPRESSION [(t1)t1.null, (t9)t9.null, ] +PREHOOK: query: select * from t7 PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@tmptable -POSTHOOK: query: insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc +PREHOOK: Input: default@t7 +#### A masked pattern was here #### +POSTHOOK: query: select * from t7 POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@tmptable -POSTHOOK: Lineage: tmptable.key EXPRESSION [] -POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src)s2.null, ] -PREHOOK: query: select * from tmptable x sort by x.key +POSTHOOK: Input: default@t7 +#### A masked pattern was here #### +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: select * from t8 PREHOOK: type: QUERY -PREHOOK: Input: default@tmptable +PREHOOK: Input: default@t8 #### A masked pattern was here #### -POSTHOOK: query: select * from tmptable x sort by x.key +POSTHOOK: query: select * from t8 POSTHOOK: type: QUERY -POSTHOOK: Input: default@tmptable +POSTHOOK: Input: default@t8 #### A masked pattern was here #### -tst1 500 -tst2 500 -PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a PREHOOK: type: QUERY -POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -14700,118 +9753,65 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: t1 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count() + keys: _col0 (type: double) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Map 4 Map Operator Tree: TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: t2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + expressions: UDFToDouble(UDFToLong(key)) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count() + keys: _col0 (type: double) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Reducer 3 Reduce Operator Tree: Group By Operator - aggregations: count(VALUE._col0) + keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst2' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -14819,219 +9819,183 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a PREHOOK: type: QUERY -PREHOOK: Input: default@src +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 #### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a POSTHOOK: type: QUERY -POSTHOOK: Input: default@src +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 #### A masked pattern was here #### -tst1 1 -tst2 1 -PREHOOK: query: drop table if exists tmptable -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@tmptable -PREHOOK: Output: default@tmptable -POSTHOOK: query: drop table if exists tmptable -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@tmptable -POSTHOOK: Output: default@tmptable -PREHOOK: query: create table tmptable(key string, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tmptable -POSTHOOK: query: create table tmptable(key string, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tmptable -PREHOOK: query: explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a PREHOOK: type: QUERY -POSTHOOK: query: explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash + alias: a + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 4 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + alias: t2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + keys: _col0 (type: double) mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Reducer 3 Reduce Operator Tree: Group By Operator - aggregations: count(VALUE._col0) + keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable - Union 3 - Vertex: Union 3 - - Stage: Stage-2 - Dependency Collection + Union 2 + Vertex: Union 2 Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable - - Stage: Stage-3 - Stats Work - Basic Stats Work: + Fetch Operator + limit: -1 + Processor Tree: + ListSink -PREHOOK: query: insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -PREHOOK: Output: default@tmptable -POSTHOOK: query: insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -POSTHOOK: Output: default@tmptable -POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from tmptable x sort by x.key, x.value +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a PREHOOK: type: QUERY -PREHOOK: Input: default@tmptable +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 #### A masked pattern was here #### -POSTHOOK: query: select * from tmptable x sort by x.key, x.value +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a POSTHOOK: type: QUERY -POSTHOOK: Input: default@tmptable +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 #### A masked pattern was here #### - - val_165 - val_193 - val_265 - val_27 - val_409 - val_484 -128 -146 val_146 -150 val_150 -213 val_213 -224 -238 val_238 -255 val_255 -273 val_273 -278 val_278 -311 val_311 -369 -401 val_401 -406 val_406 -66 val_66 -98 val_98 -tst1 500 -PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a PREHOOK: type: QUERY -POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -15042,99 +10006,100 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: t2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count() + keys: _col0 (type: double) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 5 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + alias: b + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ + key expressions: _col0 (type: string) + sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Reducer 3 Reduce Operator Tree: Group By Operator - aggregations: count(VALUE._col0) + keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -15142,46 +10107,39 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 #### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 #### A masked pattern was here #### - 7 -128 1 -146 1 -150 1 -213 1 -224 1 -238 1 -255 1 -273 1 -278 1 -311 1 -369 1 -401 1 -406 1 -66 1 -98 1 -tst1 1 -PREHOOK: query: explain - select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS STRING) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a PREHOOK: type: QUERY -POSTHOOK: query: explain - select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS STRING) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -15192,113 +10150,100 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Map 7 <- Union 4 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 + alias: a + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 4 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Map 4 Map Operator Tree: TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + alias: b + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map 5 Map Operator Tree: TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + alias: t2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToDouble(key) (type: double), key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + keys: _col0 (type: double), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: double), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -15306,336 +10251,823 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 0 +2.0 2 +4.0 4 +5.0 5 +8.0 8 +9.0 9 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(key) (type: double), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 5 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToString(CAST( _col1 AS varchar(20))) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 21 Data size: 3864 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 0 +2.0 2 +4.0 4 +5.0 5 +8.0 8 +9.0 9 +PREHOOK: query: drop table if exists test_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists test_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE test_src (key STRING, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_src +POSTHOOK: query: CREATE TABLE test_src (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_src +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 0.0) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src +POSTHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src +POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.null, ] +PREHOOK: query: SELECT COUNT(*) FROM test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src +#### A masked pattern was here #### +310 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 0.0) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src +POSTHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src +POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.null, (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) FROM test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src +#### A masked pattern was here #### +310 +PREHOOK: query: -- union34.q + +create table src10_1 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_1 +POSTHOOK: query: -- union34.q + +create table src10_1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_1 +PREHOOK: query: create table src10_2 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_2 +POSTHOOK: query: create table src10_2 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_2 +PREHOOK: query: create table src10_3 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_3 +POSTHOOK: query: create table src10_3 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_3 +PREHOOK: query: create table src10_4 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_4 +POSTHOOK: query: create table src10_4 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_4 +PREHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1 select * +insert overwrite table src10_2 select * +insert overwrite table src10_3 select * +insert overwrite table src10_4 select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src10_1 +PREHOOK: Output: default@src10_2 +PREHOOK: Output: default@src10_3 +PREHOOK: Output: default@src10_4 +POSTHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1 select * +insert overwrite table src10_2 select * +insert overwrite table src10_3 select * +insert overwrite table src10_4 select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src10_1 +POSTHOOK: Output: default@src10_2 +POSTHOOK: Output: default@src10_3 +POSTHOOK: Output: default@src10_4 +POSTHOOK: Lineage: src10_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: analyze table src10_1 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1 +PREHOOK: Output: default@src10_1 +POSTHOOK: query: analyze table src10_1 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1 +POSTHOOK: Output: default@src10_1 +PREHOOK: query: analyze table src10_2 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_2 +PREHOOK: Output: default@src10_2 +POSTHOOK: query: analyze table src10_2 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_2 +POSTHOOK: Output: default@src10_2 +PREHOOK: query: analyze table src10_3 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_3 +PREHOOK: Output: default@src10_3 +POSTHOOK: query: analyze table src10_3 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_3 +POSTHOOK: Output: default@src10_3 +PREHOOK: query: analyze table src10_4 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_4 +PREHOOK: Output: default@src10_4 +POSTHOOK: query: analyze table src10_4 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_4 +POSTHOOK: Output: default@src10_4 +PREHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 5 <- Union 6 (CONTAINS) + Map 8 <- Union 6 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src10_1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 4 + Statistics: Num rows: 11 Data size: 4048 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src10_2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: src10_3 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Map 8 + Map Operator Tree: + TableScan + alias: src10_4 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Union 2 + Vertex: Union 2 + Union 6 + Vertex: Union 6 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1 +PREHOOK: Input: default@src10_2 +PREHOOK: Input: default@src10_3 +PREHOOK: Input: default@src10_4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1 +POSTHOOK: Input: default@src10_2 +POSTHOOK: Input: default@src10_3 +POSTHOOK: Input: default@src10_4 #### A masked pattern was here #### -0 val_0 -103 val_103 -105 val_105 -116 val_116 -120 val_120 -129 val_129 -131 val_131 -136 val_136 -155 val_155 -157 val_157 -160 val_160 -180 val_180 -183 val_183 -190 val_190 -191 val_191 -193 val_193 -205 val_205 -207 val_207 -213 val_213 -223 val_223 -235 val_235 -238 val_238 -239 val_239 -24 val_24 -248 val_248 -260 val_260 -28 val_28 -286 val_286 -298 val_298 -305 val_305 -306 val_306 -307 val_307 -308 val_308 -315 val_315 -317 val_317 -321 val_321 -33 val_33 -331 val_331 -356 val_356 -360 val_360 -366 val_366 -367 val_367 -368 val_368 -37 val_37 -378 val_378 -379 val_379 -382 val_382 -389 val_389 -395 val_395 -400 val_400 -402 val_402 -406 val_406 -407 val_407 -42 val_42 -430 val_430 -436 val_436 -44 val_44 -452 val_452 -466 val_466 -467 val_467 -472 val_472 -480 val_480 -484 val_484 -485 val_485 -487 val_487 -495 val_495 -496 val_496 -498 val_498 -51 val_51 -57 val_57 -58 val_58 -65 val_65 -66 val_66 -69 val_69 -70 val_70 -78 val_78 -8 val_8 -92 val_92 -96 val_96 -104 val_104 -111 val_111 -114 val_114 -118 val_118 -12 val_12 -133 val_133 -138 val_138 -153 val_153 -156 val_156 -162 val_162 -168 val_168 -174 val_174 -176 val_176 -178 val_178 -181 val_181 -186 val_186 -187 val_187 -192 val_192 -197 val_197 -199 val_199 -2 val_2 -201 val_201 -203 val_203 -208 val_208 -214 val_214 -218 val_218 -219 val_219 -230 val_230 -237 val_237 -249 val_249 -256 val_256 -257 val_257 -277 val_277 -278 val_278 -284 val_284 -285 val_285 -310 val_310 -322 val_322 -327 val_327 -333 val_333 -336 val_336 -338 val_338 -341 val_341 -344 val_344 -348 val_348 -351 val_351 -362 val_362 -364 val_364 -374 val_374 -392 val_392 -397 val_397 -4 val_4 -403 val_403 -404 val_404 -419 val_419 -421 val_421 -427 val_427 -429 val_429 -437 val_437 -438 val_438 -439 val_439 -454 val_454 -455 val_455 -460 val_460 -463 val_463 -470 val_470 -478 val_478 -479 val_479 -489 val_489 -491 val_491 -492 val_492 -53 val_53 -67 val_67 -82 val_82 -90 val_90 -97 val_97 -10 val_10 -125 val_125 -134 val_134 -143 val_143 -145 val_145 -149 val_149 -150 val_150 -158 val_158 -163 val_163 -164 val_164 -166 val_166 -17 val_17 -170 val_170 -172 val_172 -189 val_189 -19 val_19 -195 val_195 -196 val_196 -20 val_20 -209 val_209 -216 val_216 -217 val_217 -224 val_224 -228 val_228 -229 val_229 -233 val_233 -241 val_241 -244 val_244 -247 val_247 -255 val_255 -258 val_258 -26 val_26 -263 val_263 -265 val_265 -266 val_266 -272 val_272 -273 val_273 -274 val_274 -281 val_281 -291 val_291 -296 val_296 -30 val_30 -302 val_302 -309 val_309 -316 val_316 -318 val_318 -325 val_325 -332 val_332 -335 val_335 -339 val_339 -342 val_342 -345 val_345 -353 val_353 -369 val_369 -373 val_373 -375 val_375 -377 val_377 -384 val_384 -386 val_386 -394 val_394 -396 val_396 -399 val_399 -401 val_401 -41 val_41 -413 val_413 -414 val_414 -431 val_431 -446 val_446 -448 val_448 -449 val_449 -459 val_459 -462 val_462 -468 val_468 -47 val_47 -482 val_482 -490 val_490 -493 val_493 -494 val_494 -497 val_497 -5 val_5 -54 val_54 -74 val_74 -77 val_77 -80 val_80 -84 val_84 -85 val_85 -87 val_87 -9 val_9 -95 val_95 -100 val_100 -11 val_11 -113 val_113 -119 val_119 -126 val_126 -128 val_128 -137 val_137 -146 val_146 -15 val_15 -152 val_152 165 val_165 -167 val_167 -169 val_169 -175 val_175 -177 val_177 -179 val_179 -18 val_18 -194 val_194 -200 val_200 -202 val_202 -221 val_221 -222 val_222 -226 val_226 -242 val_242 -252 val_252 -262 val_262 +238 val_238 +255 val_255 27 val_27 -275 val_275 -280 val_280 -282 val_282 -283 val_283 -287 val_287 -288 val_288 -289 val_289 -292 val_292 +278 val_278 311 val_311 -323 val_323 -34 val_34 -35 val_35 -365 val_365 -393 val_393 409 val_409 -411 val_411 -417 val_417 -418 val_418 -424 val_424 -43 val_43 -432 val_432 -435 val_435 -443 val_443 -444 val_444 -453 val_453 -457 val_457 -458 val_458 -469 val_469 -475 val_475 -477 val_477 -481 val_481 -483 val_483 -64 val_64 -72 val_72 -76 val_76 -83 val_83 +484 val_484 86 val_86 98 val_98 -PREHOOK: query: explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 PREHOOK: type: QUERY -POSTHOOK: query: explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -15646,133 +11078,141 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Map 8 <- Union 4 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Map 6 <- Union 7 (CONTAINS) + Map 9 <- Union 7 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 8 <- Union 3 (CONTAINS), Union 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash + alias: src10_1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Map 5 Map Operator Tree: TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + alias: src10_2 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: src10_3 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Map 9 Map Operator Tree: TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + alias: src10_4 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Reducer 2 Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 4048 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 + Reducer 8 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 21 Data size: 7728 Basic stats: COMPLETE Column stats: NONE + Union 3 + Vertex: Union 3 + Union 7 + Vertex: Union 7 Stage: Stage-0 Fetch Operator @@ -15780,16 +11220,35 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 PREHOOK: type: QUERY -PREHOOK: Input: default@src +PREHOOK: Input: default@src10_1 +PREHOOK: Input: default@src10_2 +PREHOOK: Input: default@src10_3 +PREHOOK: Input: default@src10_4 #### A masked pattern was here #### -POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 POSTHOOK: type: QUERY -POSTHOOK: Input: default@src +POSTHOOK: Input: default@src10_1 +POSTHOOK: Input: default@src10_2 +POSTHOOK: Input: default@src10_3 +POSTHOOK: Input: default@src10_4 #### A masked pattern was here #### -309 +165 val_165 +238 val_238 +255 val_255 +27 val_27 +278 val_278 +311 val_311 +409 val_409 +484 val_484 +86 val_86 +98 val_98 diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out index 3b3983f896..dc4e55478a 100644 --- a/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out +++ b/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out @@ -34,82 +34,6 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@u3 POSTHOOK: Lineage: u3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: u3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from u1 -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -#### A masked pattern was here #### -POSTHOOK: query: select * from u1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: select * from u2 -PREHOOK: type: QUERY -PREHOOK: Input: default@u2 -#### A masked pattern was here #### -POSTHOOK: query: select * from u2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u2 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -PREHOOK: query: select * from u3 -PREHOOK: type: QUERY -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select * from u3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 PREHOOK: query: select key, value from ( select key, value from u1 @@ -244,38 +168,6 @@ POSTHOOK: Input: default@u3 98 val_98 PREHOOK: query: select distinct * from ( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select distinct * from -( select distinct * from u1 union select key, value from u2 diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out new file mode 100644 index 0000000000..96c212477c --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out @@ -0,0 +1,2650 @@ +PREHOOK: query: explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: drop table if exists tmptable +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists tmptable +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tmptable(key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: create table tmptable(key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from tmptable x sort by x.key, x.value +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key, x.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +tst1 500 +PREHOOK: query: drop table if exists tmptable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmptable +PREHOOK: Output: default@tmptable +POSTHOOK: query: drop table if exists tmptable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmptable +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Map 7 <- Union 4 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +103 val_103 +105 val_105 +116 val_116 +120 val_120 +129 val_129 +131 val_131 +136 val_136 +155 val_155 +157 val_157 +160 val_160 +180 val_180 +183 val_183 +190 val_190 +191 val_191 +193 val_193 +205 val_205 +207 val_207 +213 val_213 +223 val_223 +235 val_235 +238 val_238 +239 val_239 +24 val_24 +248 val_248 +260 val_260 +28 val_28 +286 val_286 +298 val_298 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +315 val_315 +317 val_317 +321 val_321 +33 val_33 +331 val_331 +356 val_356 +360 val_360 +366 val_366 +367 val_367 +368 val_368 +37 val_37 +378 val_378 +379 val_379 +382 val_382 +389 val_389 +395 val_395 +400 val_400 +402 val_402 +406 val_406 +407 val_407 +42 val_42 +430 val_430 +436 val_436 +44 val_44 +452 val_452 +466 val_466 +467 val_467 +472 val_472 +480 val_480 +484 val_484 +485 val_485 +487 val_487 +495 val_495 +496 val_496 +498 val_498 +51 val_51 +57 val_57 +58 val_58 +65 val_65 +66 val_66 +69 val_69 +70 val_70 +78 val_78 +8 val_8 +92 val_92 +96 val_96 +104 val_104 +111 val_111 +114 val_114 +118 val_118 +12 val_12 +133 val_133 +138 val_138 +153 val_153 +156 val_156 +162 val_162 +168 val_168 +174 val_174 +176 val_176 +178 val_178 +181 val_181 +186 val_186 +187 val_187 +192 val_192 +197 val_197 +199 val_199 +2 val_2 +201 val_201 +203 val_203 +208 val_208 +214 val_214 +218 val_218 +219 val_219 +230 val_230 +237 val_237 +249 val_249 +256 val_256 +257 val_257 +277 val_277 +278 val_278 +284 val_284 +285 val_285 +310 val_310 +322 val_322 +327 val_327 +333 val_333 +336 val_336 +338 val_338 +341 val_341 +344 val_344 +348 val_348 +351 val_351 +362 val_362 +364 val_364 +374 val_374 +392 val_392 +397 val_397 +4 val_4 +403 val_403 +404 val_404 +419 val_419 +421 val_421 +427 val_427 +429 val_429 +437 val_437 +438 val_438 +439 val_439 +454 val_454 +455 val_455 +460 val_460 +463 val_463 +470 val_470 +478 val_478 +479 val_479 +489 val_489 +491 val_491 +492 val_492 +53 val_53 +67 val_67 +82 val_82 +90 val_90 +97 val_97 +10 val_10 +125 val_125 +134 val_134 +143 val_143 +145 val_145 +149 val_149 +150 val_150 +158 val_158 +163 val_163 +164 val_164 +166 val_166 +17 val_17 +170 val_170 +172 val_172 +189 val_189 +19 val_19 +195 val_195 +196 val_196 +20 val_20 +209 val_209 +216 val_216 +217 val_217 +224 val_224 +228 val_228 +229 val_229 +233 val_233 +241 val_241 +244 val_244 +247 val_247 +255 val_255 +258 val_258 +26 val_26 +263 val_263 +265 val_265 +266 val_266 +272 val_272 +273 val_273 +274 val_274 +281 val_281 +291 val_291 +296 val_296 +30 val_30 +302 val_302 +309 val_309 +316 val_316 +318 val_318 +325 val_325 +332 val_332 +335 val_335 +339 val_339 +342 val_342 +345 val_345 +353 val_353 +369 val_369 +373 val_373 +375 val_375 +377 val_377 +384 val_384 +386 val_386 +394 val_394 +396 val_396 +399 val_399 +401 val_401 +41 val_41 +413 val_413 +414 val_414 +431 val_431 +446 val_446 +448 val_448 +449 val_449 +459 val_459 +462 val_462 +468 val_468 +47 val_47 +482 val_482 +490 val_490 +493 val_493 +494 val_494 +497 val_497 +5 val_5 +54 val_54 +74 val_74 +77 val_77 +80 val_80 +84 val_84 +85 val_85 +87 val_87 +9 val_9 +95 val_95 +100 val_100 +11 val_11 +113 val_113 +119 val_119 +126 val_126 +128 val_128 +137 val_137 +146 val_146 +15 val_15 +152 val_152 +165 val_165 +167 val_167 +169 val_169 +175 val_175 +177 val_177 +179 val_179 +18 val_18 +194 val_194 +200 val_200 +202 val_202 +221 val_221 +222 val_222 +226 val_226 +242 val_242 +252 val_252 +262 val_262 +27 val_27 +275 val_275 +280 val_280 +282 val_282 +283 val_283 +287 val_287 +288 val_288 +289 val_289 +292 val_292 +311 val_311 +323 val_323 +34 val_34 +35 val_35 +365 val_365 +393 val_393 +409 val_409 +411 val_411 +417 val_417 +418 val_418 +424 val_424 +43 val_43 +432 val_432 +435 val_435 +443 val_443 +444 val_444 +453 val_453 +457 val_457 +458 val_458 +469 val_469 +475 val_475 +477 val_477 +481 val_481 +483 val_483 +64 val_64 +72 val_72 +76 val_76 +83 val_83 +86 val_86 +98 val_98 +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) + Reducer 7 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 1 +tst2 1 +tst3 1 +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 7 <- Union 3 (CONTAINS) + Map 8 <- Union 5 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 11 <- Union 2 (CONTAINS) + Map 12 <- Union 4 (CONTAINS) + Map 13 <- Union 6 (CONTAINS) + Map 14 <- Union 8 (CONTAINS) + Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 11 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 12 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 14 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 10 <- Union 7 (CONTAINS) + Map 9 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 6 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 8 <- Union 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 10 + Map Operator Tree: + TableScan + alias: s4 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 9 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 66 Data size: 35904 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 66 Data size: 35904 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Union 3 + Vertex: Union 3 + Union 7 + Vertex: Union 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +2 val_2 2 val_2 +4 val_4 4 val_4 +5 val_5 5 val_5 +8 val_8 8 val_8 +9 val_9 9 val_9 +tst1 500 tst1 500 +PREHOOK: query: explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 10 <- Union 2 (CONTAINS) + Map 11 <- Union 4 (CONTAINS) + Map 12 <- Union 6 (CONTAINS) + Map 13 <- Union 8 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: '1' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 42500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 10 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: reverse(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 11 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 12 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: astring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: no inputs + Map 13 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: lstring[0] (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 85 Data size: 16320 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 85 Data size: 16320 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src_thrift +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src_thrift +#### A masked pattern was here #### +03 1 +071 1 +09 1 +1 1 +10 1 +100 1 +104 1 +111 1 +114 1 +116 1 +123 1 +126 1 +128 1 +131 1 +134 1 +14 1 +152 1 +155 1 +169 1 +17 1 +170 1 +172 1 +178 1 +179 1 +182 1 +189 1 +19 1 +192 1 +194 1 +195 1 +199 1 +20 1 +207 1 +21 1 +213 1 +218 1 +223 1 +229 1 +233 1 +235 1 +24 1 +256 1 +257 1 +26 1 +265 1 +266 1 +27 1 +278 1 +280 1 +284 1 +291 1 +294 1 +30 1 +301 1 +302 1 +305 1 +312 1 +314 1 +316 1 +321 1 +323 1 +33 1 +333 1 +338 1 +344 1 +353 1 +356 1 +361 1 +364 1 +368 1 +369 1 +37 1 +377 1 +378 1 +386 1 +392 1 +397 1 +40 1 +404 1 +407 1 +411 1 +418 1 +419 1 +422 1 +424 1 +430 1 +432 1 +436 1 +437 1 +448 1 +454 1 +457 1 +46 1 +468 1 +477 1 +479 1 +490 1 +492 1 +493 1 +495 1 +496 1 +497 1 +501 1 +533 1 +534 1 +552 1 +57 1 +582 1 +611 1 +62 1 +652 1 +67 1 +671 1 +691 1 +712 1 +723 1 +73 1 +732 1 +75 1 +751 1 +761 1 +794 1 +8 1 +80 1 +814 1 +822 1 +831 1 +85 1 +854 1 +863 1 +871 1 +874 1 +882 1 +9 1 +902 1 +904 1 +91 1 +911 1 +92 1 +921 1 +922 1 +964 1 +97 1 +974 1 +982 1 +983 1 +record_7 1 +record_8 1 +NULL 1 +002 1 +02 1 +032 1 +034 1 +07 1 +074 1 +091 1 +105 1 +11 1 +113 1 +124 1 +136 1 +137 1 +138 1 +143 1 +150 1 +158 1 +160 1 +162 1 +165 1 +166 1 +174 1 +177 1 +18 1 +180 1 +186 1 +196 1 +197 1 +204 1 +217 1 +219 1 +226 1 +230 1 +238 1 +242 1 +248 1 +252 1 +260 1 +261 1 +263 1 +274 1 +277 1 +293 1 +296 1 +304 1 +307 1 +308 1 +325 1 +327 1 +342 1 +348 1 +35 1 +351 1 +367 1 +381 1 +389 1 +393 1 +394 1 +4 1 +403 1 +409 1 +412 1 +414 1 +421 1 +427 1 +429 1 +435 1 +438 1 +439 1 +45 1 +455 1 +458 1 +460 1 +461 1 +462 1 +471 1 +475 1 +478 1 +480 1 +484 1 +487 1 +50 1 +503 1 +513 1 +521 1 +523 1 +53 1 +532 1 +551 1 +56 1 +563 1 +571 1 +573 1 +58 1 +59 1 +591 1 +603 1 +604 1 +622 1 +633 1 +634 1 +641 1 +644 1 +651 1 +66 1 +661 1 +69 1 +693 1 +694 1 +703 1 +71 1 +731 1 +734 1 +752 1 +754 1 +764 1 +77 1 +772 1 +78 1 +784 1 +793 1 +803 1 +813 1 +83 1 +833 1 +843 1 +844 1 +864 1 +89 1 +90 1 +924 1 +934 1 +95 1 +954 1 +961 1 +993 1 +record_0 1 +record_2 1 +record_4 1 +record_6 1 +record_9 1 +001 1 +004 1 +01 1 +013 1 +051 1 +061 1 +062 1 +063 1 +102 1 +103 1 +120 1 +129 1 +133 1 +142 1 +145 1 +146 1 +149 1 +15 1 +157 1 +181 1 +187 1 +191 1 +2 1 +200 1 +201 1 +202 1 +203 1 +208 1 +214 1 +216 1 +222 1 +224 1 +228 1 +237 1 +239 1 +241 1 +244 1 +254 1 +255 1 +258 1 +262 1 +264 1 +273 1 +275 1 +28 1 +282 1 +285 1 +286 1 +287 1 +288 1 +29 1 +292 1 +298 1 +310 1 +311 1 +332 1 +335 1 +336 1 +339 1 +354 1 +360 1 +362 1 +373 1 +384 1 +395 1 +396 1 +402 1 +417 1 +42 1 +43 1 +442 1 +444 1 +449 1 +453 1 +459 1 +47 1 +48 1 +481 1 +482 1 +483 1 +485 1 +489 1 +494 1 +502 1 +541 1 +562 1 +574 1 +593 1 +60 1 +621 1 +631 1 +65 1 +653 1 +663 1 +664 1 +681 1 +682 1 +683 1 +713 1 +72 1 +74 1 +742 1 +76 1 +763 1 +771 1 +781 1 +79 1 +791 1 +811 1 +812 1 +82 1 +834 1 +852 1 +86 1 +861 1 +87 1 +872 1 +894 1 +903 1 +912 1 +914 1 +932 1 +942 1 +944 1 +96 1 +963 1 +record_5 1 +0 1 +021 1 +064 1 +08 1 +081 1 +082 1 +084 1 +094 1 +118 1 +119 1 +12 1 +122 1 +125 1 +153 1 +156 1 +163 1 +164 1 +167 1 +168 1 +175 1 +176 1 +183 1 +184 1 +190 1 +193 1 +205 1 +209 1 +221 1 +234 1 +243 1 +247 1 +249 1 +251 1 +271 1 +272 1 +281 1 +283 1 +289 1 +306 1 +309 1 +315 1 +317 1 +318 1 +322 1 +331 1 +34 1 +341 1 +345 1 +365 1 +366 1 +372 1 +374 1 +375 1 +379 1 +38 1 +382 1 +391 1 +399 1 +400 1 +401 1 +406 1 +41 1 +413 1 +431 1 +44 1 +443 1 +446 1 +452 1 +463 1 +466 1 +467 1 +469 1 +470 1 +472 1 +473 1 +491 1 +498 1 +5 1 +51 1 +54 1 +543 1 +554 1 +561 1 +572 1 +584 1 +594 1 +612 1 +613 1 +64 1 +662 1 +68 1 +692 1 +70 1 +702 1 +704 1 +714 1 +724 1 +773 1 +774 1 +782 1 +802 1 +81 1 +821 1 +832 1 +84 1 +842 1 +851 1 +873 1 +892 1 +933 1 +941 1 +971 1 +973 1 +98 1 +981 1 +984 1 +991 1 +record_1 1 +record_3 1 diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index a02a57c00b..bc980b6bc5 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -463,3 +463,61 @@ Stage-0 TableScan [TS_3] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] +PREHOOK: query: SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: explain analyze +SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze +SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_9] + Transform Operator [SCR_8] (rows=809/1028 width=178) + command:cat + Merge Join Operator [MERGEJOIN_14] (rows=809/1028 width=178) + Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_3] + PartitionCols:key + Filter Operator [FIL_12] (rows=500/500 width=178) + predicate:key is not null + TableScan [TS_0] (rows=500/500 width=178) + default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Map 3 [SIMPLE_EDGE] + SHUFFLE [RS_5] + PartitionCols:key + Filter Operator [FIL_13] (rows=500/500 width=87) + predicate:key is not null + TableScan [TS_1] (rows=500/500 width=87) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + diff --git a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out deleted file mode 100644 index 3b3983f896..0000000000 --- a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out +++ /dev/null @@ -1,553 +0,0 @@ -PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@u1 -POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@u1 -POSTHOOK: Lineage: u1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: u1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@u2 -POSTHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@u2 -POSTHOOK: Lineage: u2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: u2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@u3 -POSTHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@u3 -POSTHOOK: Lineage: u3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: u3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from u1 -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -#### A masked pattern was here #### -POSTHOOK: query: select * from u1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: select * from u2 -PREHOOK: type: QUERY -PREHOOK: Input: default@u2 -#### A masked pattern was here #### -POSTHOOK: query: select * from u2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u2 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -PREHOOK: query: select * from u3 -PREHOOK: type: QUERY -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select * from u3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union all -select key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union all -select key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union distinct -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union distinct -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union -select key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union -select key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: drop view if exists v -PREHOOK: type: DROPVIEW -POSTHOOK: query: drop view if exists v -POSTHOOK: type: DROPVIEW -PREHOOK: query: create view v as select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -PREHOOK: Output: database:default -PREHOOK: Output: default@v -POSTHOOK: query: create view v as select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v -POSTHOOK: Lineage: v.key EXPRESSION [(u1)u1.FieldSchema(name:key, type:string, comment:null), (u2)u2.FieldSchema(name:key, type:string, comment:null), (u3)u3.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: v.value EXPRESSION [(u1)u1.FieldSchema(name:value, type:string, comment:null), (u2)u2.FieldSchema(name:value, type:string, comment:null), (u3)u3.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: describe extended v -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@v -POSTHOOK: query: describe extended v -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@v -key string -value string - -#### A masked pattern was here #### -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select `u2`.`key`, `u2`.`value` from `default`.`u2` -union all -select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) -PREHOOK: query: select * from v -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: select * from v -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -POSTHOOK: Input: default@v -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: drop view if exists v -PREHOOK: type: DROPVIEW -PREHOOK: Input: default@v -PREHOOK: Output: default@v -POSTHOOK: query: drop view if exists v -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: default@v -POSTHOOK: Output: default@v -PREHOOK: query: create view v as select tab.* from -( -select distinct * from u1 -union -select distinct * from u2 -) tab -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Output: database:default -PREHOOK: Output: default@v -POSTHOOK: query: create view v as select tab.* from -( -select distinct * from u1 -union -select distinct * from u2 -) tab -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v -POSTHOOK: Lineage: v.key EXPRESSION [(u1)u1.FieldSchema(name:key, type:string, comment:null), (u2)u2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: v.value EXPRESSION [(u1)u1.FieldSchema(name:value, type:string, comment:null), (u2)u2.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: describe extended v -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@v -POSTHOOK: query: describe extended v -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@v -key string -value string - -#### A masked pattern was here #### -( -select distinct * from u1 -union -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) -PREHOOK: query: select * from v -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: select * from v -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@v -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: drop view if exists v -PREHOOK: type: DROPVIEW -PREHOOK: Input: default@v -PREHOOK: Output: default@v -POSTHOOK: query: drop view if exists v -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: default@v -POSTHOOK: Output: default@v -PREHOOK: query: create view v as select * from -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Output: database:default -PREHOOK: Output: default@v -POSTHOOK: query: create view v as select * from -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v -POSTHOOK: Lineage: v.key EXPRESSION [(u1)u1.FieldSchema(name:key, type:string, comment:null), (u2)u2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: v.value EXPRESSION [(u1)u1.FieldSchema(name:value, type:string, comment:null), (u2)u2.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: describe extended v -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@v -POSTHOOK: query: describe extended v -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@v -key string -value string - -#### A masked pattern was here #### -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union all -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) -PREHOOK: query: select * from v -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: select * from v -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@v -#### A masked pattern was here #### -0 val_0 -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -- 2.13.6 (Apple Git-96)