diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java index 002dafaa0c..9acce1f8bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java @@ -107,21 +107,9 @@ public Object process(Node nd, Stack stack, // operator graph. There's typically two reasons for that: a) mux/demux // b) multi insert. Mux/Demux will hit the same leaf again, multi insert // will result into a vertex with multiple FS or RS operators. - if (context.childToWorkMap.containsKey(operator)) { - // if we've seen both root and child, we can bail. - - // clear out the mapjoin set. we don't need it anymore. - context.currentMapJoinOperators.clear(); - - // clear out the union set. we don't need it anymore. - context.currentUnionOperators.clear(); - - return null; - } else { // At this point we don't have to do anything special. Just // run through the regular paces w/o creating a new task. work = context.rootToWorkMap.get(root); - } } else { // create a new vertex if (context.preceedingWork == null) { @@ -344,15 +332,12 @@ public Object process(Node nd, Stack stack, // we've not seen this terminal before. we need to check // rootUnionWorkMap which contains the information of mapping the root // operator of a union work to a union work - unionWork = context.rootUnionWorkMap.get(root); - if (unionWork == null) { // if unionWork is null, it means it is the first time. we need to // create a union work object and add this work to it. Subsequent // work should reference the union and not the actual work. unionWork = GenTezUtils.createUnionWork(context, root, operator, tezWork); // finally connect the union work with work connectUnionWorkWithWork(unionWork, work, tezWork, context); - } } context.currentUnionOperators.clear(); work = unionWork; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 4b10e8974e..216fcaf670 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -483,7 +483,7 @@ private void semijoinRemovalBasedTransformations(OptimizeTezProcContext procCtx, // Remove any semi join edges from Union Op perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); - removeSemiJoinEdgesForUnion(procCtx); + //removeSemiJoinEdgesForUnion(procCtx); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Remove any semi join edge between Union and RS"); diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 1dd9a0ff30..cb6f277718 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -2716,23 +2716,23 @@ POSTHOOK: Output: default@c_n4 Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 9 (BROADCAST_EDGE) -Map 10 <- Union 11 (CONTAINS) -Map 12 <- Union 11 (CONTAINS) -Map 14 <- Union 15 (CONTAINS) -Map 16 <- Union 15 (CONTAINS) -Map 17 <- Union 15 (CONTAINS) -Map 19 <- Map 9 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 20 <- Map 9 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 21 <- Map 9 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 22 <- Map 9 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 9 <- Map 18 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 11 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Union 3 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Union 3 (CUSTOM_SIMPLE_EDGE) -Reducer 7 <- Map 1 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 15 (SIMPLE_EDGE), Union 3 (CONTAINS) +Map 1 <- Map 14 (BROADCAST_EDGE) +Map 14 <- Map 23 (BROADCAST_EDGE) +Map 15 <- Union 16 (CONTAINS) +Map 17 <- Union 16 (CONTAINS) +Map 19 <- Union 20 (CONTAINS) +Map 21 <- Union 20 (CONTAINS) +Map 22 <- Union 20 (CONTAINS) +Map 24 <- Map 14 (BROADCAST_EDGE), Union 10 (CONTAINS), Union 3 (CONTAINS), Union 4 (CONTAINS), Union 6 (CONTAINS), Union 7 (CONTAINS), Union 9 (CONTAINS) +Map 25 <- Map 14 (BROADCAST_EDGE), Union 10 (CONTAINS), Union 3 (CONTAINS), Union 4 (CONTAINS), Union 6 (CONTAINS), Union 7 (CONTAINS), Union 9 (CONTAINS) +Map 26 <- Map 14 (BROADCAST_EDGE), Union 10 (CONTAINS), Union 3 (CONTAINS), Union 4 (CONTAINS), Union 6 (CONTAINS), Union 7 (CONTAINS), Union 9 (CONTAINS) +Map 27 <- Map 14 (BROADCAST_EDGE), Union 10 (CONTAINS), Union 3 (CONTAINS), Union 4 (CONTAINS), Union 6 (CONTAINS), Union 7 (CONTAINS), Union 9 (CONTAINS) +Reducer 11 <- Union 10 (CUSTOM_SIMPLE_EDGE) +Reducer 12 <- Map 1 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE) +Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 10 (CONTAINS), Union 20 (SIMPLE_EDGE), Union 3 (CONTAINS), Union 4 (CONTAINS), Union 6 (CONTAINS), Union 7 (CONTAINS), Union 9 (CONTAINS) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 10 (CONTAINS), Union 16 (SIMPLE_EDGE), Union 3 (CONTAINS), Union 4 (CONTAINS), Union 6 (CONTAINS), Union 7 (CONTAINS), Union 9 (CONTAINS) +Reducer 5 <- Union 4 (CUSTOM_SIMPLE_EDGE) +Reducer 8 <- Union 7 (CUSTOM_SIMPLE_EDGE) Stage-5 Stats Work{} @@ -2742,24 +2742,24 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 - Reducer 4 llap - File Output Operator [FS_82] - Group By Operator [GBY_80] (rows=1 width=880) + Reducer 11 llap + File Output Operator [FS_100] + Group By Operator [GBY_98] (rows=1 width=880) Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] - <-Union 3 [CUSTOM_SIMPLE_EDGE] - <-Map 19 [CONTAINS] llap + <-Union 10 [CUSTOM_SIMPLE_EDGE] + <-Map 24 [CONTAINS] llap File Output Operator [FS_278] table:{"name:":"default.a_n19"} Select Operator [SEL_275] (rows=1677 width=10) Output:["_col0","_col1"] Map Join Operator [MAPJOIN_273] (rows=1677 width=10) Conds:RS_357._col1=SEL_274._col0(Inner),Output:["_col0","_col3"] - <-Map 9 [BROADCAST_EDGE] vectorized, llap + <-Map 14 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_357] PartitionCols:_col1 Map Join Operator [MAPJOIN_356] (rows=27 width=7) Conds:SEL_354._col0=RS_352._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 18 [BROADCAST_EDGE] vectorized, llap + <-Map 23 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_352] PartitionCols:_col0 Select Operator [SEL_351] (rows=25 width=7) @@ -2804,14 +2804,14 @@ Stage-5 Select Operator [SEL_283] (rows=3409 width=10) Output:["key","value"] Please refer to the previous Select Operator [SEL_275] - <-Map 20 [CONTAINS] llap + <-Map 25 [CONTAINS] llap File Output Operator [FS_298] table:{"name:":"default.a_n19"} Select Operator [SEL_295] (rows=1677 width=10) Output:["_col0","_col1"] Map Join Operator [MAPJOIN_293] (rows=1677 width=10) Conds:RS_358._col1=SEL_294._col0(Inner),Output:["_col0","_col3"] - <-Map 9 [BROADCAST_EDGE] vectorized, llap + <-Map 14 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_358] PartitionCols:_col1 Please refer to the previous Map Join Operator [MAPJOIN_356] @@ -2845,14 +2845,14 @@ Stage-5 Select Operator [SEL_303] (rows=3409 width=10) Output:["key","value"] Please refer to the previous Select Operator [SEL_295] - <-Map 21 [CONTAINS] llap + <-Map 26 [CONTAINS] llap File Output Operator [FS_318] table:{"name:":"default.a_n19"} Select Operator [SEL_315] (rows=1677 width=10) Output:["_col0","_col1"] Map Join Operator [MAPJOIN_313] (rows=1677 width=10) Conds:RS_359._col1=SEL_314._col0(Inner),Output:["_col0","_col3"] - <-Map 9 [BROADCAST_EDGE] vectorized, llap + <-Map 14 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_359] PartitionCols:_col1 Please refer to the previous Map Join Operator [MAPJOIN_356] @@ -2886,14 +2886,14 @@ Stage-5 Select Operator [SEL_323] (rows=3409 width=10) Output:["key","value"] Please refer to the previous Select Operator [SEL_315] - <-Map 22 [CONTAINS] llap + <-Map 27 [CONTAINS] llap File Output Operator [FS_338] table:{"name:":"default.a_n19"} Select Operator [SEL_335] (rows=1677 width=10) Output:["_col0","_col1"] Map Join Operator [MAPJOIN_333] (rows=1677 width=10) Conds:RS_360._col1=SEL_334._col0(Inner),Output:["_col0","_col3"] - <-Map 9 [BROADCAST_EDGE] vectorized, llap + <-Map 14 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_360] PartitionCols:_col1 Please refer to the previous Map Join Operator [MAPJOIN_356] @@ -2927,79 +2927,14 @@ Stage-5 Select Operator [SEL_343] (rows=3409 width=10) Output:["key","value"] Please refer to the previous Select Operator [SEL_335] - <-Reducer 2 [CONTAINS] llap - File Output Operator [FS_218] - table:{"name:":"default.a_n19"} - Select Operator [SEL_216] (rows=605 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_215] (rows=605 width=10) - Conds:RS_367._col3=Union 11._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_367] - PartitionCols:_col3 - Map Join Operator [MAPJOIN_365] (rows=550 width=10) - Conds:SEL_363._col0=RS_355._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 9 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_355] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_354] - <-Select Operator [SEL_363] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_361] (rows=500 width=10) - predicate:key is not null - TableScan [TS_0] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Union 11 [SIMPLE_EDGE] - <-Map 10 [CONTAINS] vectorized, llap - Reduce Output Operator [RS_373] - PartitionCols:_col0 - Select Operator [SEL_372] (rows=25 width=7) - Output:["_col0"] - Filter Operator [FIL_371] (rows=25 width=7) - predicate:value is not null - TableScan [TS_245] (rows=25 width=7) - Output:["value"] - <-Map 12 [CONTAINS] vectorized, llap - Reduce Output Operator [RS_376] - PartitionCols:_col0 - Select Operator [SEL_375] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_374] (rows=500 width=10) - predicate:value is not null - TableScan [TS_250] (rows=500 width=10) - Output:["value"] - Reduce Output Operator [RS_227] - Group By Operator [GBY_224] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] - Select Operator [SEL_219] (rows=3409 width=10) - Output:["key","value"] - Please refer to the previous Select Operator [SEL_216] - File Output Operator [FS_220] - table:{"name:":"default.b_n15"} - Please refer to the previous Select Operator [SEL_216] - Reduce Output Operator [RS_228] - Group By Operator [GBY_225] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] - Select Operator [SEL_221] (rows=3409 width=10) - Output:["key","value"] - Please refer to the previous Select Operator [SEL_216] - File Output Operator [FS_222] - table:{"name:":"default.c_n4"} - Please refer to the previous Select Operator [SEL_216] - Reduce Output Operator [RS_229] - Group By Operator [GBY_226] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] - Select Operator [SEL_223] (rows=3409 width=10) - Output:["key","value"] - Please refer to the previous Select Operator [SEL_216] - <-Reducer 8 [CONTAINS] llap + <-Reducer 13 [CONTAINS] llap File Output Operator [FS_233] table:{"name:":"default.a_n19"} Select Operator [SEL_231] (rows=1127 width=10) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_230] (rows=1127 width=10) - Conds:RS_41._col1=Union 15._col0(Inner),Output:["_col0","_col3"] - <-Reducer 7 [SIMPLE_EDGE] llap + Conds:RS_41._col1=Union 20._col0(Inner),Output:["_col0","_col3"] + <-Reducer 12 [SIMPLE_EDGE] llap SHUFFLE [RS_41] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_204] (rows=550 width=10) @@ -3011,8 +2946,9 @@ Stage-5 Output:["_col0","_col1"] Filter Operator [FIL_362] (rows=500 width=10) predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_0] - <-Map 13 [SIMPLE_EDGE] vectorized, llap + TableScan [TS_0] (rows=500 width=10) + default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 18 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_370] PartitionCols:_col0 Select Operator [SEL_369] (rows=500 width=10) @@ -3021,8 +2957,8 @@ Stage-5 predicate:key is not null TableScan [TS_24] (rows=500 width=10) default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Union 15 [SIMPLE_EDGE] - <-Map 14 [CONTAINS] vectorized, llap + <-Union 20 [SIMPLE_EDGE] + <-Map 19 [CONTAINS] vectorized, llap Reduce Output Operator [RS_379] PartitionCols:_col0 Select Operator [SEL_378] (rows=25 width=7) @@ -3031,7 +2967,7 @@ Stage-5 predicate:value is not null TableScan [TS_255] (rows=25 width=7) Output:["value"] - <-Map 16 [CONTAINS] vectorized, llap + <-Map 21 [CONTAINS] vectorized, llap Reduce Output Operator [RS_382] PartitionCols:_col0 Select Operator [SEL_381] (rows=500 width=10) @@ -3040,7 +2976,7 @@ Stage-5 predicate:value is not null TableScan [TS_260] (rows=500 width=10) Output:["value"] - <-Map 17 [CONTAINS] vectorized, llap + <-Map 22 [CONTAINS] vectorized, llap Reduce Output Operator [RS_385] PartitionCols:_col0 Select Operator [SEL_384] (rows=500 width=10) @@ -3073,16 +3009,293 @@ Stage-5 Select Operator [SEL_238] (rows=3409 width=10) Output:["key","value"] Please refer to the previous Select Operator [SEL_231] + <-Reducer 2 [CONTAINS] llap + File Output Operator [FS_218] + table:{"name:":"default.a_n19"} + Select Operator [SEL_216] (rows=605 width=10) + Output:["_col0","_col1"] + Merge Join Operator [MERGEJOIN_215] (rows=605 width=10) + Conds:RS_367._col3=Union 16._col0(Inner),Output:["_col1","_col2"] + <-Map 1 [SIMPLE_EDGE] vectorized, llap + SHUFFLE [RS_367] + PartitionCols:_col3 + Map Join Operator [MAPJOIN_365] (rows=550 width=10) + Conds:SEL_363._col0=RS_355._col0(Inner),Output:["_col1","_col2","_col3"] + <-Map 14 [BROADCAST_EDGE] vectorized, llap + BROADCAST [RS_355] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_354] + <-Select Operator [SEL_363] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_361] (rows=500 width=10) + predicate:key is not null + Please refer to the previous TableScan [TS_0] + <-Union 16 [SIMPLE_EDGE] + <-Map 15 [CONTAINS] vectorized, llap + Reduce Output Operator [RS_373] + PartitionCols:_col0 + Select Operator [SEL_372] (rows=25 width=7) + Output:["_col0"] + Filter Operator [FIL_371] (rows=25 width=7) + predicate:value is not null + TableScan [TS_245] (rows=25 width=7) + Output:["value"] + <-Map 17 [CONTAINS] vectorized, llap + Reduce Output Operator [RS_376] + PartitionCols:_col0 + Select Operator [SEL_375] (rows=500 width=10) + Output:["_col0"] + Filter Operator [FIL_374] (rows=500 width=10) + predicate:value is not null + TableScan [TS_250] (rows=500 width=10) + Output:["value"] + Reduce Output Operator [RS_227] + Group By Operator [GBY_224] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] + Select Operator [SEL_219] (rows=3409 width=10) + Output:["key","value"] + Please refer to the previous Select Operator [SEL_216] + File Output Operator [FS_220] + table:{"name:":"default.b_n15"} + Please refer to the previous Select Operator [SEL_216] + Reduce Output Operator [RS_228] + Group By Operator [GBY_225] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] + Select Operator [SEL_221] (rows=3409 width=10) + Output:["key","value"] + Please refer to the previous Select Operator [SEL_216] + File Output Operator [FS_222] + table:{"name:":"default.c_n4"} + Please refer to the previous Select Operator [SEL_216] + Reduce Output Operator [RS_229] + Group By Operator [GBY_226] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] + Select Operator [SEL_223] (rows=3409 width=10) + Output:["key","value"] + Please refer to the previous Select Operator [SEL_216] Reducer 5 llap + File Output Operator [FS_82] + Group By Operator [GBY_80] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] + <-Union 4 [CUSTOM_SIMPLE_EDGE] + <-Map 24 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_278] + Please refer to the previous Reduce Output Operator [RS_287] + Please refer to the previous File Output Operator [FS_280] + Please refer to the previous Reduce Output Operator [RS_288] + Please refer to the previous File Output Operator [FS_282] + Please refer to the previous Reduce Output Operator [RS_289] + <-Map 25 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_298] + Please refer to the previous Reduce Output Operator [RS_307] + Please refer to the previous File Output Operator [FS_300] + Please refer to the previous Reduce Output Operator [RS_308] + Please refer to the previous File Output Operator [FS_302] + Please refer to the previous Reduce Output Operator [RS_309] + <-Map 26 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_318] + Please refer to the previous Reduce Output Operator [RS_327] + Please refer to the previous File Output Operator [FS_320] + Please refer to the previous Reduce Output Operator [RS_328] + Please refer to the previous File Output Operator [FS_322] + Please refer to the previous Reduce Output Operator [RS_329] + <-Map 27 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_338] + Please refer to the previous Reduce Output Operator [RS_347] + Please refer to the previous File Output Operator [FS_340] + Please refer to the previous Reduce Output Operator [RS_348] + Please refer to the previous File Output Operator [FS_342] + Please refer to the previous Reduce Output Operator [RS_349] + <-Reducer 13 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_233] + Please refer to the previous Reduce Output Operator [RS_242] + Please refer to the previous File Output Operator [FS_235] + Please refer to the previous Reduce Output Operator [RS_243] + Please refer to the previous File Output Operator [FS_237] + Please refer to the previous Reduce Output Operator [RS_244] + <-Reducer 2 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_218] + Please refer to the previous Reduce Output Operator [RS_227] + Please refer to the previous File Output Operator [FS_220] + Please refer to the previous Reduce Output Operator [RS_228] + Please refer to the previous File Output Operator [FS_222] + Please refer to the previous Reduce Output Operator [RS_229] + Reducer 8 llap File Output Operator [FS_91] Group By Operator [GBY_89] (rows=1 width=880) Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] - <- Please refer to the previous Union 3 [CUSTOM_SIMPLE_EDGE] - Reducer 6 llap - File Output Operator [FS_100] - Group By Operator [GBY_98] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] - <- Please refer to the previous Union 3 [CUSTOM_SIMPLE_EDGE] + <-Union 7 [CUSTOM_SIMPLE_EDGE] + <-Map 24 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_278] + Please refer to the previous Reduce Output Operator [RS_287] + Please refer to the previous File Output Operator [FS_280] + Please refer to the previous Reduce Output Operator [RS_288] + Please refer to the previous File Output Operator [FS_282] + Please refer to the previous Reduce Output Operator [RS_289] + <-Map 25 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_298] + Please refer to the previous Reduce Output Operator [RS_307] + Please refer to the previous File Output Operator [FS_300] + Please refer to the previous Reduce Output Operator [RS_308] + Please refer to the previous File Output Operator [FS_302] + Please refer to the previous Reduce Output Operator [RS_309] + <-Map 26 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_318] + Please refer to the previous Reduce Output Operator [RS_327] + Please refer to the previous File Output Operator [FS_320] + Please refer to the previous Reduce Output Operator [RS_328] + Please refer to the previous File Output Operator [FS_322] + Please refer to the previous Reduce Output Operator [RS_329] + <-Map 27 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_338] + Please refer to the previous Reduce Output Operator [RS_347] + Please refer to the previous File Output Operator [FS_340] + Please refer to the previous Reduce Output Operator [RS_348] + Please refer to the previous File Output Operator [FS_342] + Please refer to the previous Reduce Output Operator [RS_349] + <-Reducer 13 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_233] + Please refer to the previous Reduce Output Operator [RS_242] + Please refer to the previous File Output Operator [FS_235] + Please refer to the previous Reduce Output Operator [RS_243] + Please refer to the previous File Output Operator [FS_237] + Please refer to the previous Reduce Output Operator [RS_244] + <-Reducer 2 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_218] + Please refer to the previous Reduce Output Operator [RS_227] + Please refer to the previous File Output Operator [FS_220] + Please refer to the previous Reduce Output Operator [RS_228] + Please refer to the previous File Output Operator [FS_222] + Please refer to the previous Reduce Output Operator [RS_229] + Union 3 + <-Map 24 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_278] + Please refer to the previous Reduce Output Operator [RS_287] + Please refer to the previous File Output Operator [FS_280] + Please refer to the previous Reduce Output Operator [RS_288] + Please refer to the previous File Output Operator [FS_282] + Please refer to the previous Reduce Output Operator [RS_289] + <-Map 25 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_298] + Please refer to the previous Reduce Output Operator [RS_307] + Please refer to the previous File Output Operator [FS_300] + Please refer to the previous Reduce Output Operator [RS_308] + Please refer to the previous File Output Operator [FS_302] + Please refer to the previous Reduce Output Operator [RS_309] + <-Map 26 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_318] + Please refer to the previous Reduce Output Operator [RS_327] + Please refer to the previous File Output Operator [FS_320] + Please refer to the previous Reduce Output Operator [RS_328] + Please refer to the previous File Output Operator [FS_322] + Please refer to the previous Reduce Output Operator [RS_329] + <-Map 27 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_338] + Please refer to the previous Reduce Output Operator [RS_347] + Please refer to the previous File Output Operator [FS_340] + Please refer to the previous Reduce Output Operator [RS_348] + Please refer to the previous File Output Operator [FS_342] + Please refer to the previous Reduce Output Operator [RS_349] + <-Reducer 13 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_233] + Please refer to the previous Reduce Output Operator [RS_242] + Please refer to the previous File Output Operator [FS_235] + Please refer to the previous Reduce Output Operator [RS_243] + Please refer to the previous File Output Operator [FS_237] + Please refer to the previous Reduce Output Operator [RS_244] + <-Reducer 2 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_218] + Please refer to the previous Reduce Output Operator [RS_227] + Please refer to the previous File Output Operator [FS_220] + Please refer to the previous Reduce Output Operator [RS_228] + Please refer to the previous File Output Operator [FS_222] + Please refer to the previous Reduce Output Operator [RS_229] + Union 6 + <-Map 24 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_278] + Please refer to the previous Reduce Output Operator [RS_287] + Please refer to the previous File Output Operator [FS_280] + Please refer to the previous Reduce Output Operator [RS_288] + Please refer to the previous File Output Operator [FS_282] + Please refer to the previous Reduce Output Operator [RS_289] + <-Map 25 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_298] + Please refer to the previous Reduce Output Operator [RS_307] + Please refer to the previous File Output Operator [FS_300] + Please refer to the previous Reduce Output Operator [RS_308] + Please refer to the previous File Output Operator [FS_302] + Please refer to the previous Reduce Output Operator [RS_309] + <-Map 26 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_318] + Please refer to the previous Reduce Output Operator [RS_327] + Please refer to the previous File Output Operator [FS_320] + Please refer to the previous Reduce Output Operator [RS_328] + Please refer to the previous File Output Operator [FS_322] + Please refer to the previous Reduce Output Operator [RS_329] + <-Map 27 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_338] + Please refer to the previous Reduce Output Operator [RS_347] + Please refer to the previous File Output Operator [FS_340] + Please refer to the previous Reduce Output Operator [RS_348] + Please refer to the previous File Output Operator [FS_342] + Please refer to the previous Reduce Output Operator [RS_349] + <-Reducer 13 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_233] + Please refer to the previous Reduce Output Operator [RS_242] + Please refer to the previous File Output Operator [FS_235] + Please refer to the previous Reduce Output Operator [RS_243] + Please refer to the previous File Output Operator [FS_237] + Please refer to the previous Reduce Output Operator [RS_244] + <-Reducer 2 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_218] + Please refer to the previous Reduce Output Operator [RS_227] + Please refer to the previous File Output Operator [FS_220] + Please refer to the previous Reduce Output Operator [RS_228] + Please refer to the previous File Output Operator [FS_222] + Please refer to the previous Reduce Output Operator [RS_229] + Union 9 + <-Map 24 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_278] + Please refer to the previous Reduce Output Operator [RS_287] + Please refer to the previous File Output Operator [FS_280] + Please refer to the previous Reduce Output Operator [RS_288] + Please refer to the previous File Output Operator [FS_282] + Please refer to the previous Reduce Output Operator [RS_289] + <-Map 25 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_298] + Please refer to the previous Reduce Output Operator [RS_307] + Please refer to the previous File Output Operator [FS_300] + Please refer to the previous Reduce Output Operator [RS_308] + Please refer to the previous File Output Operator [FS_302] + Please refer to the previous Reduce Output Operator [RS_309] + <-Map 26 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_318] + Please refer to the previous Reduce Output Operator [RS_327] + Please refer to the previous File Output Operator [FS_320] + Please refer to the previous Reduce Output Operator [RS_328] + Please refer to the previous File Output Operator [FS_322] + Please refer to the previous Reduce Output Operator [RS_329] + <-Map 27 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_338] + Please refer to the previous Reduce Output Operator [RS_347] + Please refer to the previous File Output Operator [FS_340] + Please refer to the previous Reduce Output Operator [RS_348] + Please refer to the previous File Output Operator [FS_342] + Please refer to the previous Reduce Output Operator [RS_349] + <-Reducer 13 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_233] + Please refer to the previous Reduce Output Operator [RS_242] + Please refer to the previous File Output Operator [FS_235] + Please refer to the previous Reduce Output Operator [RS_243] + Please refer to the previous File Output Operator [FS_237] + Please refer to the previous Reduce Output Operator [RS_244] + <-Reducer 2 [CONTAINS] llap + Please refer to the previous File Output Operator [FS_218] + Please refer to the previous Reduce Output Operator [RS_227] + Please refer to the previous File Output Operator [FS_220] + Please refer to the previous Reduce Output Operator [RS_228] + Please refer to the previous File Output Operator [FS_222] + Please refer to the previous Reduce Output Operator [RS_229] Stage-6 Stats Work{} Stage-1 @@ -3725,13 +3938,13 @@ POSTHOOK: Output: default@dest2_n43 Plan optimized by CBO. Vertex dependency in root stage -Map 8 <- Union 3 (CONTAINS) -Map 9 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +Map 10 <- Union 3 (CONTAINS), Union 6 (CONTAINS) +Map 9 <- Union 3 (CONTAINS), Union 6 (CONTAINS) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS), Union 6 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Union 3 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +Reducer 7 <- Union 6 (SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) Stage-4 Stats Work{} @@ -3753,21 +3966,7 @@ Stage-4 Group By Operator [GBY_15] (rows=1 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_52] - PartitionCols:_col0 - Group By Operator [GBY_50] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_48] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_47] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1 - Group By Operator [GBY_51] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_48] - <-Map 9 [CONTAINS] llap + <-Map 10 [CONTAINS] llap Reduce Output Operator [RS_59] PartitionCols:_col0 Group By Operator [GBY_57] (rows=1 width=280) @@ -3781,6 +3980,20 @@ Stage-4 Group By Operator [GBY_58] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) Please refer to the previous Select Operator [SEL_55] + <-Map 9 [CONTAINS] llap + Reduce Output Operator [RS_52] + PartitionCols:_col0 + Group By Operator [GBY_50] (rows=1 width=280) + Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) + Select Operator [SEL_48] (rows=500 width=10) + Output:["_col0","_col1"] + TableScan [TS_47] (rows=500 width=10) + Output:["key","value"] + Reduce Output Operator [RS_53] + PartitionCols:_col0, _col1 + Group By Operator [GBY_51] (rows=1 width=464) + Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) + Please refer to the previous Select Operator [SEL_48] <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_45] PartitionCols:_col0 @@ -3808,18 +4021,27 @@ Stage-4 Select Operator [SEL_20] (rows=1 width=272) Output:["key","value"] Please refer to the previous Select Operator [SEL_16] - Reducer 7 llap + Reducer 8 llap File Output Operator [FS_39] Group By Operator [GBY_37] (rows=1 width=1320) Output:["_col0","_col1","_col2"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)","compute_stats(VALUE._col2)"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap + <-Reducer 7 [CUSTOM_SIMPLE_EDGE] llap File Output Operator [FS_32] table:{"name:":"default.dest2_n43"} Select Operator [SEL_30] (rows=1 width=456) Output:["_col0","_col1","_col2"] Group By Operator [GBY_29] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + <-Union 6 [SIMPLE_EDGE] + <-Map 10 [CONTAINS] llap + Please refer to the previous Reduce Output Operator [RS_59] + Please refer to the previous Reduce Output Operator [RS_60] + <-Map 9 [CONTAINS] llap + Please refer to the previous Reduce Output Operator [RS_52] + Please refer to the previous Reduce Output Operator [RS_53] + <-Reducer 2 [CONTAINS] llap + Please refer to the previous Reduce Output Operator [RS_45] + Please refer to the previous Reduce Output Operator [RS_46] PARTITION_ONLY_SHUFFLE [RS_36] Group By Operator [GBY_35] (rows=1 width=1320) Output:["_col0","_col1","_col2"],aggregations:["compute_stats(key, 'hll')","compute_stats(val1, 'hll')","compute_stats(val2, 'hll')"] @@ -3862,12 +4084,12 @@ POSTHOOK: Output: default@dest2_n43 Plan optimized by CBO. Vertex dependency in root stage -Map 8 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +Map 9 <- Union 3 (CONTAINS), Union 6 (CONTAINS) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS), Union 6 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Union 3 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +Reducer 7 <- Union 6 (SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) Stage-4 Stats Work{} @@ -3889,7 +4111,7 @@ Stage-4 Group By Operator [GBY_13] (rows=1 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] llap + <-Map 9 [CONTAINS] llap Reduce Output Operator [RS_50] PartitionCols:_col0 Group By Operator [GBY_48] (rows=1 width=280) @@ -3930,18 +4152,24 @@ Stage-4 Select Operator [SEL_18] (rows=1 width=272) Output:["key","value"] Please refer to the previous Select Operator [SEL_14] - Reducer 7 llap + Reducer 8 llap File Output Operator [FS_37] Group By Operator [GBY_35] (rows=1 width=1320) Output:["_col0","_col1","_col2"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)","compute_stats(VALUE._col2)"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap + <-Reducer 7 [CUSTOM_SIMPLE_EDGE] llap File Output Operator [FS_30] table:{"name:":"default.dest2_n43"} Select Operator [SEL_28] (rows=1 width=456) Output:["_col0","_col1","_col2"] Group By Operator [GBY_27] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + <-Union 6 [SIMPLE_EDGE] + <-Map 9 [CONTAINS] llap + Please refer to the previous Reduce Output Operator [RS_50] + Please refer to the previous Reduce Output Operator [RS_51] + <-Reducer 2 [CONTAINS] llap + Please refer to the previous Reduce Output Operator [RS_43] + Please refer to the previous Reduce Output Operator [RS_44] PARTITION_ONLY_SHUFFLE [RS_34] Group By Operator [GBY_33] (rows=1 width=1320) Output:["_col0","_col1","_col2"],aggregations:["compute_stats(key, 'hll')","compute_stats(val1, 'hll')","compute_stats(val2, 'hll')"] diff --git a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out index 05d259b0d9..556a222158 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out @@ -59,13 +59,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 8 <- Union 3 (CONTAINS) - Map 9 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Map 10 <- Union 3 (CONTAINS), Union 6 (CONTAINS) + Map 9 <- Union 3 (CONTAINS), Union 6 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS), Union 6 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Union 3 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) + Reducer 7 <- Union 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -86,10 +86,10 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs - Map 8 + Map 10 Map Operator Tree: TableScan - alias: s2 + alias: s0 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -122,7 +122,7 @@ STAGE PLANS: Map 9 Map Operator Tree: TableScan - alias: s0 + alias: s2 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -235,7 +235,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -269,7 +269,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Reducer 7 + Reducer 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -286,6 +286,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 3 Vertex: Union 3 + Union 6 + Vertex: Union 6 Stage: Stage-3 Dependency Collection @@ -1040,13 +1042,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 9 <- Union 2 (CONTAINS) + Map 1 <- Union 2 (CONTAINS), Union 5 (CONTAINS) + Map 10 <- Union 2 (CONTAINS), Union 5 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) + Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS), Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 @@ -1082,25 +1084,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Map 9 + Map 10 Map Operator Tree: TableScan alias: s2 @@ -1133,6 +1117,24 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -1182,7 +1184,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Reducer 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1216,7 +1218,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Reducer 6 + Reducer 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1231,7 +1233,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Reducer 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1267,6 +1269,8 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Union 2 Vertex: Union 2 + Union 5 + Vertex: Union 5 Stage: Stage-3 Dependency Collection @@ -2017,13 +2021,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 9 <- Union 2 (CONTAINS) + Map 1 <- Union 2 (CONTAINS), Union 5 (CONTAINS) + Map 10 <- Union 2 (CONTAINS), Union 5 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) + Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS), Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 @@ -2059,25 +2063,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Map 9 + Map 10 Map Operator Tree: TableScan alias: s2 @@ -2110,6 +2096,24 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -2159,7 +2163,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Reducer 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2193,7 +2197,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Reducer 6 + Reducer 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2208,7 +2212,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Reducer 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2244,6 +2248,8 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Union 2 Vertex: Union 2 + Union 5 + Vertex: Union 5 Stage: Stage-3 Dependency Collection @@ -2986,12 +2992,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 8 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Map 9 <- Union 3 (CONTAINS), Union 6 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS), Union 6 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Union 3 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) + Reducer 7 <- Union 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3012,7 +3018,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs - Map 8 + Map 9 Map Operator Tree: TableScan alias: s2 @@ -3128,7 +3134,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3162,7 +3168,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Reducer 7 + Reducer 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3179,6 +3185,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 3 Vertex: Union 3 + Union 6 + Vertex: Union 6 Stage: Stage-3 Dependency Collection