From 20cd9e07634a800b00d0f6dab98c1611c1e08a5c Mon Sep 17 00:00:00 2001 From: Jesus Camacho Rodriguez Date: Sun, 3 May 2020 00:43:24 -0700 Subject: [PATCH] HIVE-23298 : Disable RS deduplication step in Optimizer if it is run in TezCompiler Signed-off-by: Ashutosh Chauhan --- .../hive/ql/optimizer/ConvertJoinMapJoin.java | 12 +- .../hadoop/hive/ql/optimizer/Optimizer.java | 3 +- .../clientpositive/llap/auto_join18.q.out | 8 +- .../llap/auto_join18_multi_distinct.q.out | 8 +- .../results/clientpositive/llap/bucket2.q.out | 46 ++- .../results/clientpositive/llap/bucket4.q.out | 46 ++- .../llap/bucket_num_reducers2.q.out | 50 ++- .../llap/check_constraint.q.out | 42 +-- .../llap/correlationoptimizer14.q.out | 178 ++++++++--- .../llap/disable_merge_for_bucketing.q.out | 46 ++- .../llap/distinct_groupby.q.out | 4 +- .../clientpositive/llap/distinct_stats.q.out | 6 +- .../llap/dynpart_sort_optimization2.q.out | 12 +- .../llap/enforce_constraint_notnull.q.out | 67 ++-- .../clientpositive/llap/except_all.q.out | 46 +-- .../clientpositive/llap/except_distinct.q.out | 68 ++-- .../clientpositive/llap/explainuser_1.q.out | 50 +-- .../clientpositive/llap/explainuser_2.q.out | 2 +- .../clientpositive/llap/groupby3_map.q.out | 20 +- .../llap/groupby3_map_multi_distinct.q.out | 20 +- .../llap/groupby3_map_skew.q.out | 20 +- .../clientpositive/llap/groupby4_map.q.out | 20 +- .../llap/groupby4_map_skew.q.out | 20 +- .../clientpositive/llap/groupby5_map.q.out | 20 +- .../llap/groupby5_map_skew.q.out | 20 +- .../llap/insert_into_default_keyword.q.out | 21 +- .../clientpositive/llap/intersect_all.q.out | 6 +- .../llap/intersect_distinct.q.out | 6 +- .../clientpositive/llap/limit_pushdown.q.out | 279 ++++++++++------- .../results/clientpositive/llap/mrr.q.out | 16 +- .../llap/offset_limit_ppd_optimizer.q.out | 293 ++++++++++-------- .../results/clientpositive/llap/ptf.q.out | 93 +++--- .../llap/reduce_deduplicate.q.out | 46 ++- .../llap/reduce_deduplicate_extended.q.out | 58 ++-- .../llap/reducesink_dedup.q.out | 1 - .../clientpositive/llap/subquery_ANY.q.out | 12 +- .../clientpositive/llap/subquery_in.q.out | 68 ++-- .../clientpositive/llap/subquery_notin.q.out | 28 +- .../clientpositive/llap/subquery_scalar.q.out | 12 +- .../clientpositive/llap/subquery_select.q.out | 138 ++++----- .../results/clientpositive/llap/tez_dml.q.out | 20 +- .../clientpositive/llap/tez_union2.q.out | 16 +- .../llap/tez_union_multiinsert.q.out | 2 +- .../clientpositive/llap/unionDistinct_1.q.out | 59 ++-- .../clientpositive/llap/unionDistinct_3.q.out | 12 +- .../llap/vector_decimal_6.q.out | 20 +- .../llap/vector_groupby_reduce.q.out | 20 +- .../vector_outer_reference_windowed.q.out | 126 ++++---- .../clientpositive/llap/vector_ptf_1.q.out | 61 ++-- .../llap/vector_windowing.q.out | 61 ++-- .../llap/vectorization_limit.q.out | 35 ++- .../clientpositive/llap/vectorized_ptf.q.out | 93 +++--- .../perf/tez/constraints/query51.q.out | 164 +++++----- .../perf/tez/constraints/query53.q.out | 4 +- .../perf/tez/constraints/query63.q.out | 4 +- .../clientpositive/perf/tez/query51.q.out | 164 +++++----- .../clientpositive/perf/tez/query53.q.out | 4 +- .../clientpositive/perf/tez/query63.q.out | 4 +- 58 files changed, 1474 insertions(+), 1306 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index d8d8cae936..3207f390ca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -767,7 +767,7 @@ private boolean checkConvertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcCont for (Operator parentOp : joinOp.getParentOperators()) { if (!(parentOp instanceof ReduceSinkOperator)) { // could be mux/demux operators. Currently not supported - LOG.info("Found correlation optimizer operators. Cannot convert to SMB at this time."); + LOG.debug("Found correlation optimizer operators. Cannot convert to SMB at this time."); return false; } ReduceSinkOperator rsOp = (ReduceSinkOperator) parentOp; @@ -788,14 +788,20 @@ private boolean checkConvertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcCont return false; } } + // check Parent's traits are same as rs + OpTraits parentTraits = rsOp.getParentOperators().get(0).getOpTraits(); + if (null == parentTraits) { + // programming error - shouldn't be null + return false; + } - if (!checkColEquality(rsOp.getParentOperators().get(0).getOpTraits().getSortCols(), rsOp + if (!checkColEquality(parentTraits.getSortCols(), rsOp .getOpTraits().getSortCols(), rsOp.getColumnExprMap(), false)) { LOG.info("We cannot convert to SMB because the sort column names do not match."); return false; } - if (!checkColEquality(rsOp.getParentOperators().get(0).getOpTraits().getBucketColNames(), rsOp + if (!checkColEquality(parentTraits.getBucketColNames(), rsOp .getOpTraits().getBucketColNames(), rsOp.getColumnExprMap(), true)) { LOG.info("We cannot convert to SMB because bucket column names do not match."); return false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index da277d058f..07bd6e722b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -191,7 +191,8 @@ public void initialize(HiveConf hiveConf) { transformations.add(new FixedBucketPruningOptimizer(compatMode)); } - if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION)) { + if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION) && + !isTezExecEngine) { transformations.add(new ReduceSinkDeDuplication()); } transformations.add(new NonBlockingOpDeDupProc()); diff --git a/ql/src/test/results/clientpositive/llap/auto_join18.q.out b/ql/src/test/results/clientpositive/llap/auto_join18.q.out index 505ce8ca5b..865806c25c 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join18.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join18.q.out @@ -119,11 +119,11 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 274 Data size: 2457 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 262 Data size: 1323 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 274 Data size: 2457 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 262 Data size: 1323 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col0) minReductionHashAggr: 0.99 @@ -163,13 +163,13 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Stage: Stage-0 diff --git a/ql/src/test/results/clientpositive/llap/auto_join18_multi_distinct.q.out b/ql/src/test/results/clientpositive/llap/auto_join18_multi_distinct.q.out index c77bed7072..113ed6a61c 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join18_multi_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join18_multi_distinct.q.out @@ -121,11 +121,11 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 282 Data size: 3349 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 274 Data size: 2561 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0,_col1,_col2,_col3,_col4) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 282 Data size: 3349 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 274 Data size: 2561 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col0) minReductionHashAggr: 0.99 @@ -169,13 +169,13 @@ STAGE PLANS: keys: _col2 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 16 Data size: 1632 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 1224 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 16 Data size: 1632 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 1224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Stage: Stage-0 diff --git a/ql/src/test/results/clientpositive/llap/bucket2.q.out b/ql/src/test/results/clientpositive/llap/bucket2.q.out index 9b82a96fc1..18fa16ad75 100644 --- a/ql/src/test/results/clientpositive/llap/bucket2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -158,33 +158,29 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:struct - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/bucket4.q.out b/ql/src/test/results/clientpositive/llap/bucket4.q.out index ea9dc76a3a..fd9a442491 100644 --- a/ql/src/test/results/clientpositive/llap/bucket4.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -159,33 +159,29 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:struct - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out b/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out index 17f30f9e58..6926517f2a 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out @@ -156,33 +156,29 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:struct - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/check_constraint.q.out b/ql/src/test/results/clientpositive/llap/check_constraint.q.out index e4fe16427f..e896b6925e 100644 --- a/ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -2496,15 +2496,18 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint((_col2 is not null and ((_col1 > 0) and ((_col1 < 100) or (_col1 = 5))) is not false)) (type: boolean) Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.tmerge - Write Type: UPDATE + Select Operator + expressions: _col0 (type: struct), _col1 (type: int), '1' (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.tmerge + Write Type: UPDATE Stage: Stage-4 Dependency Collection @@ -2746,15 +2749,18 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint((_col2 is not null and ((_col1 > 0) and ((_col1 < 100) or (_col1 = 5))) is not false)) (type: boolean) Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.tmerge - Write Type: UPDATE + Select Operator + expressions: _col0 (type: struct), _col1 (type: int), '1' (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.tmerge + Write Type: UPDATE Reducer 6 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out index defd2bdb99..eb6420f2e2 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out @@ -1181,17 +1181,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Map 1 <- Reducer 5 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: x - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_15_y_key_min) AND DynamicValue(RS_15_y_key_max) and in_bloom_filter(key, DynamicValue(RS_15_y_key_bloom_filter))) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and key BETWEEN DynamicValue(RS_15_y_key_min) AND DynamicValue(RS_15_y_key_max) and in_bloom_filter(key, DynamicValue(RS_15_y_key_bloom_filter))) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -1206,7 +1209,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 + Map 4 Map Operator Tree: TableScan alias: y @@ -1232,6 +1235,39 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1239,27 +1275,30 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000000) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Stage: Stage-0 Fetch Operator @@ -1355,17 +1394,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Map 1 <- Reducer 5 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: x - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_15_y_key_min) AND DynamicValue(RS_15_y_key_max) and in_bloom_filter(key, DynamicValue(RS_15_y_key_bloom_filter))) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and key BETWEEN DynamicValue(RS_15_y_key_min) AND DynamicValue(RS_15_y_key_max) and in_bloom_filter(key, DynamicValue(RS_15_y_key_bloom_filter))) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -1380,7 +1422,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 + Map 4 Map Operator Tree: TableScan alias: y @@ -1406,6 +1448,39 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1413,27 +1488,30 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 18 Data size: 4896 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000000) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index 389a5f2769..af0835d518 100644 --- a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -158,33 +158,29 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:struct - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/distinct_groupby.q.out b/ql/src/test/results/clientpositive/llap/distinct_groupby.q.out index b396e454c7..8796d15234 100644 --- a/ql/src/test/results/clientpositive/llap/distinct_groupby.q.out +++ b/ql/src/test/results/clientpositive/llap/distinct_groupby.q.out @@ -57,10 +57,10 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/distinct_stats.q.out b/ql/src/test/results/clientpositive/llap/distinct_stats.q.out index f0daa4c4f1..c3c55e8029 100644 --- a/ql/src/test/results/clientpositive/llap/distinct_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/distinct_stats.q.out @@ -84,14 +84,14 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 11875 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index bb3b6c39f0..1fa5c46b4f 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -1272,14 +1272,14 @@ STAGE PLANS: keys: ss_sold_date_sk (type: int) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 1704 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1768 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 1704 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1768 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 1704 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1768 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1536,14 +1536,14 @@ STAGE PLANS: keys: ss_sold_date_sk (type: int) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 1704 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1768 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 1704 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1768 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 1704 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1768 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out index 6abd6f3c82..d4d3c21572 100644 --- a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out +++ b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out @@ -3417,7 +3417,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 567 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE @@ -3516,7 +3516,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 189 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE @@ -4523,15 +4523,18 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(_col1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.masking_test_n4 - Write Type: UPDATE + Select Operator + expressions: _col0 (type: struct), _col1 (type: int), '1' (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test_n4 + Write Type: UPDATE Stage: Stage-4 Dependency Collection @@ -4820,15 +4823,18 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(_col1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.masking_test_n4 - Write Type: UPDATE + Select Operator + expressions: _col0 (type: struct), _col1 (type: int), '1' (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test_n4 + Write Type: UPDATE Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -5399,15 +5405,18 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(_col1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.masking_test_n4 - Write Type: UPDATE + Select Operator + expressions: _col0 (type: struct), _col1 (type: int), '1' (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 409 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test_n4 + Write Type: UPDATE Reducer 6 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/except_all.q.out b/ql/src/test/results/clientpositive/llap/except_all.q.out index 4c2498f5a8..1edefb2874 100644 --- a/ql/src/test/results/clientpositive/llap/except_all.q.out +++ b/ql/src/test/results/clientpositive/llap/except_all.q.out @@ -526,20 +526,20 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 270 Data size: 52380 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 260 Data size: 50440 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 13 Execution mode: vectorized, llap @@ -689,28 +689,28 @@ STAGE PLANS: keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1860 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), 2L (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 270 Data size: 52380 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 260 Data size: 50440 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 9 Execution mode: vectorized, llap @@ -720,17 +720,17 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col2 > 0L) and ((_col2 * 2L) = _col3)) (type: boolean) - Statistics: Num rows: 22 Data size: 4268 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 4074 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 22 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3738 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3738 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -839,28 +839,28 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), 2L (type: bigint), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: bigint), (_col1 * _col2) (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col1), sum(_col2) keys: _col0 (type: int) minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 4 Execution mode: vectorized, llap @@ -870,7 +870,7 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col1 > 0L) and ((_col1 * 2L) = _col2)) (type: boolean) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE @@ -898,28 +898,28 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), 1L (type: bigint), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: bigint), (_col1 * _col2) (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col1), sum(_col2) keys: _col0 (type: int) minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Union 3 Vertex: Union 3 diff --git a/ql/src/test/results/clientpositive/llap/except_distinct.q.out b/ql/src/test/results/clientpositive/llap/except_distinct.q.out index 47f45c5cdd..e84e62831c 100644 --- a/ql/src/test/results/clientpositive/llap/except_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/except_distinct.q.out @@ -481,20 +481,20 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 274 Data size: 53156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 261 Data size: 50634 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 12 Execution mode: vectorized, llap @@ -579,28 +579,28 @@ STAGE PLANS: keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 41 Data size: 7626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), 2L (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 41 Data size: 7954 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 270 Data size: 52380 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 6 Execution mode: vectorized, llap @@ -610,41 +610,41 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col2 > 0L) and ((_col2 * 2L) = _col3)) (type: boolean) - Statistics: Num rows: 24 Data size: 4656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 4268 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 4656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 4268 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 24 Data size: 4464 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2046 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), 2L (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24 Data size: 4656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2134 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 274 Data size: 53156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 261 Data size: 50634 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 8 Execution mode: vectorized, llap @@ -654,17 +654,17 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 130 Data size: 25220 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col2 > 0L) and ((_col2 * 2L) = _col3)) (type: boolean) - Statistics: Num rows: 22 Data size: 4268 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 4074 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 22 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3738 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3738 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -685,20 +685,20 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 270 Data size: 52380 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 135 Data size: 26190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Union 3 Vertex: Union 3 @@ -804,28 +804,28 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), 2L (type: bigint), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: bigint), (_col1 * _col2) (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col1), sum(_col2) keys: _col0 (type: int) minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 4 Execution mode: vectorized, llap @@ -835,7 +835,7 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col1 > 0L) and ((_col1 * 2L) = _col2)) (type: boolean) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE @@ -863,28 +863,28 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), 1L (type: bigint), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: bigint), (_col1 * _col2) (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col1), sum(_col2) keys: _col0 (type: int) minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Union 3 Vertex: Union 3 diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 7f0ce5a9c7..65b59aa9d6 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -2097,7 +2097,7 @@ Stage-0 Output:["_col0","_col1"] Filter Operator [FIL_17] (rows=500 width=178) predicate:_col2 is null - Merge Join Operator [MERGEJOIN_27] (rows=635 width=178) + Merge Join Operator [MERGEJOIN_27] (rows=566 width=178) Conds:RS_14._col1=RS_15._col1(Left Outer),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_14] @@ -2109,9 +2109,9 @@ Stage-0 <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_15] PartitionCols:_col1 - Select Operator [SEL_13] (rows=83 width=95) + Select Operator [SEL_13] (rows=41 width=95) Output:["_col0","_col1"] - Group By Operator [GBY_12] (rows=83 width=91) + Group By Operator [GBY_12] (rows=41 width=91) Output:["_col0"],keys:_col1 Select Operator [SEL_8] (rows=83 width=178) Output:["_col1"] @@ -5010,27 +5010,29 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] PTF Operator [PTF_11] (rows=26 width=223) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS LAST","partition by:":"_col0"}] - Group By Operator [GBY_8] (rows=26 width=223) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=26 width=223) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:rand() - Select Operator [SEL_4] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS LAST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Select Operator [SEL_15] + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_8] (rows=26 width=223) + Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Reducer 3 [SIMPLE_EDGE] llap + SHUFFLE [RS_7] + PartitionCols:_col0 + Group By Operator [GBY_6] (rows=26 width=223) + Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Reducer 2 [SIMPLE_EDGE] llap + SHUFFLE [RS_5] + PartitionCols:rand() + Select Operator [SEL_4] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + PTF Operator [PTF_3] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS LAST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] PREHOOK: query: explain select abc.* diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 6f275c6ecf..241ae87fb6 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -3612,7 +3612,7 @@ Stage-4 Output:["key","val1","val2"] Select Operator [SEL_33] (rows=1 width=456) Output:["_col0","_col1","_col2"] - Group By Operator [GBY_32] (rows=1 width=459) + Group By Operator [GBY_32] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_13] Stage-5 diff --git a/ql/src/test/results/clientpositive/llap/groupby3_map.q.out b/ql/src/test/results/clientpositive/llap/groupby3_map.q.out index 0bef509f54..93972862f3 100644 --- a/ql/src/test/results/clientpositive/llap/groupby3_map.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby3_map.q.out @@ -102,18 +102,14 @@ STAGE PLANS: aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll'), compute_stats(c6, 'hll'), compute_stats(c7, 'hll'), compute_stats(c8, 'hll'), compute_stats(c9, 'hll') mode: complete outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 3816 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 3816 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 3816 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 3960 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3960 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out b/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out index 2290c22814..25526614bb 100644 --- a/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out @@ -106,18 +106,14 @@ STAGE PLANS: aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll'), compute_stats(c6, 'hll'), compute_stats(c7, 'hll'), compute_stats(c8, 'hll'), compute_stats(c9, 'hll'), compute_stats(c10, 'hll'), compute_stats(c11, 'hll') mode: complete outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 4664 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 4664 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4664 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 4840 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4840 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/groupby3_map_skew.q.out b/ql/src/test/results/clientpositive/llap/groupby3_map_skew.q.out index 258e54591e..761cea91a6 100644 --- a/ql/src/test/results/clientpositive/llap/groupby3_map_skew.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby3_map_skew.q.out @@ -117,18 +117,14 @@ STAGE PLANS: aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll'), compute_stats(c6, 'hll'), compute_stats(c7, 'hll'), compute_stats(c8, 'hll'), compute_stats(c9, 'hll') mode: complete outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 3816 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 3816 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 3816 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 3960 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3960 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/groupby4_map.q.out b/ql/src/test/results/clientpositive/llap/groupby4_map.q.out index 7d4d7a0524..dc5c5eb7e2 100644 --- a/ql/src/test/results/clientpositive/llap/groupby4_map.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby4_map.q.out @@ -78,18 +78,14 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll') mode: complete outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/groupby4_map_skew.q.out b/ql/src/test/results/clientpositive/llap/groupby4_map_skew.q.out index eb53e25441..14cd4909e7 100644 --- a/ql/src/test/results/clientpositive/llap/groupby4_map_skew.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby4_map_skew.q.out @@ -78,18 +78,14 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll') mode: complete outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/groupby5_map.q.out b/ql/src/test/results/clientpositive/llap/groupby5_map.q.out index ddd0557df2..aa6b0908cd 100644 --- a/ql/src/test/results/clientpositive/llap/groupby5_map.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby5_map.q.out @@ -80,18 +80,14 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll') mode: complete outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/groupby5_map_skew.q.out b/ql/src/test/results/clientpositive/llap/groupby5_map_skew.q.out index b6d681b3f1..c21e2de3df 100644 --- a/ql/src/test/results/clientpositive/llap/groupby5_map_skew.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby5_map_skew.q.out @@ -80,18 +80,14 @@ STAGE PLANS: aggregations: compute_stats(key, 'hll') mode: complete outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out b/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out index b7355fb2d2..bf8270de8c 100644 --- a/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out @@ -3065,15 +3065,18 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(_col1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.acidtable - Write Type: UPDATE + Select Operator + expressions: _col0 (type: struct), _col1 (type: int), 'a1' (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acidtable + Write Type: UPDATE Reducer 7 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/intersect_all.q.out b/ql/src/test/results/clientpositive/llap/intersect_all.q.out index 549cca487a..7b24c10816 100644 --- a/ql/src/test/results/clientpositive/llap/intersect_all.q.out +++ b/ql/src/test/results/clientpositive/llap/intersect_all.q.out @@ -1648,11 +1648,11 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col1), count(_col1) keys: _col0 (type: int) - minReductionHashAggr: 0.6666666 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE @@ -1710,7 +1710,7 @@ STAGE PLANS: Group By Operator aggregations: min(_col1), count(_col1) keys: _col0 (type: int) - minReductionHashAggr: 0.6666666 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out b/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out index 950bc4b68c..8d42dd6706 100644 --- a/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out @@ -1250,11 +1250,11 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(_col1) keys: _col0 (type: int) - minReductionHashAggr: 0.6666666 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE @@ -1305,7 +1305,7 @@ STAGE PLANS: Group By Operator aggregations: count(_col1) keys: _col0 (type: int) - minReductionHashAggr: 0.6666666 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index 63e524d5d4..50d66ac8a7 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -257,16 +257,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 24750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -474,16 +477,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 5528 Data size: 21816 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Limit + Number of rows: 20 Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -546,22 +552,28 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double) - outputColumnNames: ctinyint, cdouble + Top N Key Operator + sort order: + + keys: ctinyint (type: tinyint) + null sort order: z Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: ctinyint (type: tinyint), cdouble (type: double) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + top n: 20 + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: ctinyint, cdouble + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: ctinyint (type: tinyint), cdouble (type: double) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -571,23 +583,26 @@ STAGE PLANS: keys: KEY._col0 (type: tinyint), KEY._col1 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(_col1) keys: _col0 (type: tinyint) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 131 Data size: 1444 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 131 Data size: 1312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -650,22 +665,28 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double) - outputColumnNames: ctinyint, cdouble + Top N Key Operator + sort order: + + keys: ctinyint (type: tinyint) + null sort order: z Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: ctinyint (type: tinyint), cdouble (type: double) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + top n: 20 + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: ctinyint, cdouble + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: ctinyint (type: tinyint), cdouble (type: double) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -675,23 +696,26 @@ STAGE PLANS: keys: KEY._col0 (type: tinyint), KEY._col1 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(_col1) keys: _col0 (type: tinyint) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 131 Data size: 1444 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 131 Data size: 1312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -754,22 +778,28 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 1779850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), cstring1 (type: string), cstring2 (type: string) - outputColumnNames: _col0, _col1, _col2 + Top N Key Operator + sort order: + + keys: ctinyint (type: tinyint) + null sort order: z Statistics: Num rows: 12288 Data size: 1779850 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), 0L (type: bigint) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - null sort order: zzzz - sort order: ++++ - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE + top n: 20 + Select Operator + expressions: ctinyint (type: tinyint), cstring1 (type: string), cstring2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 12288 Data size: 1779850 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), 0L (type: bigint) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + null sort order: zzzz + sort order: ++++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -779,27 +809,30 @@ STAGE PLANS: keys: KEY._col0 (type: tinyint), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: CASE WHEN (((_col3 = 1L) and _col1 is not null)) THEN (1) ELSE (null) END (type: int), CASE WHEN (((_col3 = 2L) and _col2 is not null)) THEN (1) ELSE (null) END (type: int), _col0 (type: tinyint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(_col0), count(_col1) keys: _col2 (type: tinyint) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 131 Data size: 2492 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 131 Data size: 2360 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1084,19 +1117,22 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 2 Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -1219,16 +1255,19 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 307 Data size: 30393 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/mrr.q.out b/ql/src/test/results/clientpositive/llap/mrr.q.out index 628f91af1e..098b30b68f 100644 --- a/ql/src/test/results/clientpositive/llap/mrr.q.out +++ b/ql/src/test/results/clientpositive/llap/mrr.q.out @@ -498,22 +498,22 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: bigint), _col0 (type: string) null sort order: zz sort order: ++ - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE Reducer 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -935,22 +935,22 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: bigint), _col0 (type: string) null sort order: zz sort order: ++ - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE Reducer 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 197 Data size: 18715 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index 208646bba1..267deefe5a 100644 --- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -259,17 +259,20 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 24750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Offset of rows: 10 - Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 + Offset of rows: 10 Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -478,17 +481,20 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 5528 Data size: 21816 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Offset of rows: 10 - Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Limit + Number of rows: 20 + Offset of rows: 10 Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -551,22 +557,28 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double) - outputColumnNames: ctinyint, cdouble + Top N Key Operator + sort order: + + keys: ctinyint (type: tinyint) + null sort order: z Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: ctinyint (type: tinyint), cdouble (type: double) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + top n: 30 + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: ctinyint, cdouble + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: ctinyint (type: tinyint), cdouble (type: double) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -576,24 +588,27 @@ STAGE PLANS: keys: KEY._col0 (type: tinyint), KEY._col1 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(_col1) keys: _col0 (type: tinyint) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 131 Data size: 1444 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Offset of rows: 10 - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 131 Data size: 1312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 + Offset of rows: 10 + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -656,22 +671,28 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double) - outputColumnNames: ctinyint, cdouble + Top N Key Operator + sort order: + + keys: ctinyint (type: tinyint) + null sort order: z Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: ctinyint (type: tinyint), cdouble (type: double) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + top n: 30 + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: ctinyint, cdouble + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: ctinyint (type: tinyint), cdouble (type: double) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -681,24 +702,27 @@ STAGE PLANS: keys: KEY._col0 (type: tinyint), KEY._col1 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6144 Data size: 36368 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(_col1) keys: _col0 (type: tinyint) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 131 Data size: 1444 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Offset of rows: 10 - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 131 Data size: 1312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 + Offset of rows: 10 + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -761,22 +785,28 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 1779850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), cstring1 (type: string), cstring2 (type: string) - outputColumnNames: _col0, _col1, _col2 + Top N Key Operator + sort order: + + keys: ctinyint (type: tinyint) + null sort order: z Statistics: Num rows: 12288 Data size: 1779850 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), 0L (type: bigint) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - null sort order: zzzz - sort order: ++++ - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE + top n: 30 + Select Operator + expressions: ctinyint (type: tinyint), cstring1 (type: string), cstring2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 12288 Data size: 1779850 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), 0L (type: bigint) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + null sort order: zzzz + sort order: ++++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -786,28 +816,31 @@ STAGE PLANS: keys: KEY._col0 (type: tinyint), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: CASE WHEN (((_col3 = 1L) and _col1 is not null)) THEN (1) ELSE (null) END (type: int), CASE WHEN (((_col3 = 2L) and _col2 is not null)) THEN (1) ELSE (null) END (type: int), _col0 (type: tinyint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 24576 Data size: 3756114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24576 Data size: 2547682 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(_col0), count(_col1) keys: _col2 (type: tinyint) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 131 Data size: 2492 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Offset of rows: 10 - Statistics: Num rows: 20 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 131 Data size: 2360 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Limit + Number of rows: 20 + Offset of rows: 10 + Statistics: Num rows: 20 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1049,17 +1082,20 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 307 Data size: 30393 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Offset of rows: 10 - Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 + Offset of rows: 10 Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1496,20 +1532,23 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Offset of rows: 10 - Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 20 + Offset of rows: 10 Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Reducer 3 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/ptf.q.out b/ql/src/test/results/clientpositive/llap/ptf.q.out index c678e64902..cd10396a7d 100644 --- a/ql/src/test/results/clientpositive/llap/ptf.q.out +++ b/ql/src/test/results/clientpositive/llap/ptf.q.out @@ -979,52 +979,55 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: int - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col1 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - window function definition - alias: dense_rank_window_1 - arguments: _col1 - name: dense_rank - window function: GenericUDAFDenseRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - window function definition - alias: lag_window_2 - arguments: _col2, 1, _col2 - name: lag - window function: GenericUDAFLagEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), _col2 (type: int), (_col2 - lag_window_2) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 13 Data size: 3107 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: dense_rank_window_1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: lag_window_2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), _col2 (type: int), (_col2 - lag_window_2) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 13 Data size: 3107 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 3107 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out index 9df57473f4..9d3f0b7540 100644 --- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out +++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out @@ -160,32 +160,28 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: hdfs://### HDFS PATH ### + NumFilesPerFileSink: 1 Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 - directory: hdfs://### HDFS PATH ### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Stats Publishing Key Prefix: hdfs://### HDFS PATH ### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:struct - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out index d15ea89888..1f982914cf 100644 --- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out +++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out @@ -47,10 +47,10 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -112,14 +112,14 @@ STAGE PLANS: keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col2 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -181,14 +181,14 @@ STAGE PLANS: keys: _col0 (type: string), _col1 (type: double) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col2 (type: double), _col1 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -254,13 +254,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -548,10 +551,10 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 10875 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 10875 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -619,10 +622,10 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 11875 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 11875 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2856,13 +2859,16 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out b/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out index 84c8223214..d7ed6f0a35 100644 --- a/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out +++ b/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out @@ -121,7 +121,6 @@ STAGE PLANS: Select Operator expressions: UDFToInteger(_col0) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out b/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out index 8fa69c5aaf..09b26e2c26 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out @@ -1383,14 +1383,14 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col2 (type: string) outputColumnNames: _col0, _col2, _col3, _col5, _col6 - Statistics: Num rows: 37 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), (((_col0 >= _col2) is true and (_col3 is null or _col5) is not true) or (_col6 is true and null and (_col3 is null or _col5) is not true and (_col0 >= _col2) is not true) or ((_col0 >= _col2) and (_col3 is null or _col5) is not true and (_col0 >= _col2) is not true and _col6 is not true)) (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 37 Data size: 296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 37 Data size: 296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1413,17 +1413,17 @@ STAGE PLANS: keys: _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 13 Data size: 1612 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), true (type: boolean), _col0 (type: string), (_col2 = 0L) (type: boolean), (_col2 > _col3) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: boolean), _col3 (type: boolean), _col4 (type: boolean) Stage: Stage-0 diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out index 60522c838b..21c209a4c9 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -4558,20 +4558,20 @@ STAGE PLANS: 0 _col4 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 40 Data size: 24880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 19864 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(CASE WHEN (_col10 is null) THEN (0L) ELSE (_col10) END, true) > 0) (type: boolean) - Statistics: Num rows: 13 Data size: 8095 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6214 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6190 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: string), UDFToLong(_col5) (type: bigint) null sort order: zz sort order: ++ Map-reduce partition columns: _col4 (type: string), UDFToLong(_col5) (type: bigint) - Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap @@ -4604,13 +4604,13 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 6 Execution mode: vectorized, llap @@ -5106,20 +5106,20 @@ STAGE PLANS: 0 _col4 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 40 Data size: 24880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 19864 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(CASE WHEN (_col10 is null) THEN (0L) ELSE (_col10) END, true) > 0) (type: boolean) - Statistics: Num rows: 13 Data size: 8095 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6214 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6190 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap @@ -5131,13 +5131,13 @@ STAGE PLANS: 0 _col4 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11 - Statistics: Num rows: 13 Data size: 8255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6350 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: string), UDFToLong(_col5) (type: bigint) null sort order: zz sort order: ++ Map-reduce partition columns: _col4 (type: string), UDFToLong(_col5) (type: bigint) - Statistics: Num rows: 13 Data size: 8255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) Reducer 4 Execution mode: llap @@ -5149,17 +5149,17 @@ STAGE PLANS: 0 _col4 (type: string), UDFToLong(_col5) (type: bigint) 1 _col2 (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col13 - Statistics: Num rows: 13 Data size: 8307 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 10839 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col10 is null or (_col10 = 0L) or (_col13 is not null or _col5 is null or (_col11 < _col10)) is not true) (type: boolean) - Statistics: Num rows: 7 Data size: 4473 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 7013 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 7 Data size: 4333 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 6809 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 4333 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 6809 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -5177,13 +5177,13 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 7 Execution mode: vectorized, llap @@ -5199,13 +5199,13 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 8 Execution mode: vectorized, llap @@ -5368,20 +5368,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 39 Data size: 24253 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 19864 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(CASE WHEN (_col10 is null) THEN (0L) ELSE (_col10) END, true) > 0) (type: boolean) - Statistics: Num rows: 13 Data size: 8087 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6214 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6190 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap @@ -5393,13 +5393,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11 - Statistics: Num rows: 13 Data size: 8255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6350 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), UDFToDouble(_col5) (type: double) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), UDFToDouble(_col5) (type: double) - Statistics: Num rows: 13 Data size: 8255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 6350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) Reducer 4 Execution mode: llap @@ -5411,17 +5411,17 @@ STAGE PLANS: 0 _col0 (type: int), UDFToDouble(_col5) (type: double) 1 _col2 (type: int), _col0 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col13 - Statistics: Num rows: 13 Data size: 8307 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 10839 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col10 is null or (_col10 = 0L) or (_col13 is not null or _col5 is null or (_col11 < _col10)) is not true) (type: boolean) - Statistics: Num rows: 7 Data size: 4473 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 7013 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 7 Data size: 4333 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 6809 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 4333 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 6809 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -5439,13 +5439,13 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 8 Execution mode: vectorized, llap @@ -5465,13 +5465,13 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Filter Operator predicate: (_col2 is not null and UDFToDouble(_col1) is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 3bb3a042a0..7ccae6437d 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -1016,20 +1016,20 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4 - Statistics: Num rows: 33 Data size: 7527 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 36 Data size: 8116 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(CASE WHEN (_col4 is null) THEN (0L) ELSE (_col4) END, true) > 0) (type: boolean) - Statistics: Num rows: 11 Data size: 2517 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 2708 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 11 Data size: 2453 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 11 Data size: 2453 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: int) Reducer 3 Execution mode: llap @@ -1041,13 +1041,13 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Statistics: Num rows: 11 Data size: 2629 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 2868 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col2 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 11 Data size: 2629 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 2868 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) Reducer 4 Execution mode: llap @@ -1059,17 +1059,17 @@ STAGE PLANS: 0 _col1 (type: string), _col2 (type: int) 1 _col2 (type: string), _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col7 - Statistics: Num rows: 15 Data size: 3605 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 3844 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col4 is null or (_col4 = 0L) or (_col7 is not null or _col2 is null or (_col5 < _col4)) is not true) (type: boolean) - Statistics: Num rows: 11 Data size: 2645 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 3844 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 11 Data size: 2453 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 3568 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 2453 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 3568 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1218,13 +1218,13 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 8 Execution mode: vectorized, llap @@ -1240,13 +1240,13 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 9 Execution mode: vectorized, llap diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index 8fab16789b..d179df7d3e 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -6189,13 +6189,13 @@ STAGE PLANS: 0 _col4 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 30 Data size: 18570 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 17332 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 30 Data size: 18570 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 17332 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap @@ -6233,20 +6233,20 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(_col1) <= 1) (type: boolean) - Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Reducer 7 Execution mode: vectorized, llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out index 311cee743d..ed5f43f699 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -291,13 +291,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 - Statistics: Num rows: 40 Data size: 4720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 3696 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 40 Data size: 4720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 3696 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean), _col4 (type: bigint), _col5 (type: boolean), _col6 (type: boolean) Reducer 3 Execution mode: llap @@ -309,14 +309,14 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: int) 1 _col2 (type: string), _col0 (type: int) outputColumnNames: _col1, _col2, _col4, _col5, _col6, _col8 - Statistics: Num rows: 48 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), ((_col8 is not null and (_col5 or _col4 is null) is not true) or ((_col2 or _col6) is true and null and (_col5 or _col4 is null) is not true and _col8 is null)) (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 48 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 48 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -335,17 +335,17 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), (_col1 = 0L) (type: boolean), (_col2 < _col1) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: boolean) Reducer 5 Execution mode: vectorized, llap @@ -736,13 +736,13 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 3 Execution mode: llap @@ -754,20 +754,20 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 40 Data size: 4440 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 3512 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(CASE WHEN (_col3 is null) THEN (0L) ELSE (_col3) END, true) > 0) (type: boolean) - Statistics: Num rows: 13 Data size: 1452 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col1 is null (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: boolean) Reducer 4 Execution mode: llap @@ -779,13 +779,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 - Statistics: Num rows: 13 Data size: 1664 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1280 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int), _col0 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col1 (type: int), _col0 (type: string) - Statistics: Num rows: 13 Data size: 1664 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1280 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean), _col4 (type: bigint), _col5 (type: boolean), _col6 (type: boolean) Reducer 5 Execution mode: llap @@ -797,7 +797,7 @@ STAGE PLANS: 0 _col1 (type: int), _col0 (type: string) 1 _col0 (type: int), _col2 (type: string) outputColumnNames: _col1, _col2, _col4, _col5, _col6, _col8 - Statistics: Num rows: 13 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), ((_col5 or _col4 is null) is true or ((_col2 or _col6) is true and null and (_col5 or _col4 is null) is not true and _col8 is null) or ((_col5 or _col4 is null) is not true and _col8 is null and (_col2 or _col6) is not true)) (type: boolean) outputColumnNames: _col0, _col1 @@ -846,17 +846,17 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), (_col1 = 0L) (type: boolean), (_col2 < _col1) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: boolean) Stage: Stage-0 @@ -3332,13 +3332,13 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8 - Statistics: Num rows: 40 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 7696 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col2 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 40 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 7696 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col3 (type: boolean), _col4 (type: boolean), _col6 (type: bigint), _col7 (type: boolean), _col8 (type: boolean) Reducer 3 Execution mode: llap @@ -3350,13 +3350,13 @@ STAGE PLANS: 0 _col1 (type: string), _col2 (type: int) 1 _col2 (type: string), _col0 (type: int) outputColumnNames: _col0, _col2, _col3, _col4, _col6, _col7, _col8, _col10 - Statistics: Num rows: 48 Data size: 6852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 7279 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 48 Data size: 6852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 7279 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: boolean), _col4 (type: boolean), _col6 (type: bigint), _col7 (type: boolean), _col8 (type: boolean), _col10 (type: boolean) Reducer 4 Execution mode: llap @@ -3368,11 +3368,11 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col2, _col3, _col4, _col6, _col7, _col8, _col10, _col13 - Statistics: Num rows: 49 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 53 Data size: 1184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 49 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 53 Data size: 1184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col3 (type: boolean), _col4 (type: boolean), _col6 (type: bigint), _col7 (type: boolean), _col8 (type: boolean), _col10 (type: boolean), _col13 (type: boolean) Reducer 5 Execution mode: llap @@ -3384,14 +3384,14 @@ STAGE PLANS: 0 1 outputColumnNames: _col2, _col3, _col4, _col6, _col7, _col8, _col10, _col13, _col14, _col15 - Statistics: Num rows: 49 Data size: 1476 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 53 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: int), (((_col10 is not null and (_col7 or _col6 is null) is not true) or ((_col3 or _col8) is true and null and (_col7 or _col6 is null) is not true and _col10 is null)) and ((_col13 is not null and _col14) or ((_col4 or _col15) and null and _col14 and _col13 is null))) (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 49 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 53 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 49 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 53 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3410,17 +3410,17 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), (_col1 = 0L) (type: boolean), (_col2 < _col1) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: boolean) Filter Operator predicate: _col1 is not null (type: boolean) @@ -4519,14 +4519,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col2 (type: int) outputColumnNames: _col1, _col3, _col4 - Statistics: Num rows: 32 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 252 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), CASE WHEN (_col4 is null) THEN (0L) ELSE (_col3) END (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 32 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4586,17 +4586,17 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: bigint), true (type: boolean), _col0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: boolean) Stage: Stage-0 @@ -5284,13 +5284,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 30 Data size: 3360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 3136 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col2 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 30 Data size: 3360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 3136 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) Reducer 5 Execution mode: llap @@ -5302,13 +5302,13 @@ STAGE PLANS: 0 _col1 (type: string), _col2 (type: int) 1 _col0 (type: string), _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 30 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 32 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 30 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -5318,10 +5318,10 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 32 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 29 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.96875 + minReductionHashAggr: 0.9655172 mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -5356,20 +5356,20 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(_col1) <= 1) (type: boolean) - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -5621,20 +5621,20 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(_col1) <= 1) (type: boolean) - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reducer 12 Execution mode: llap Reduce Operator Tree: @@ -5645,13 +5645,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 30 Data size: 3360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 3136 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col2 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 30 Data size: 3360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 3136 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) Reducer 13 Execution mode: llap @@ -5663,13 +5663,13 @@ STAGE PLANS: 0 _col1 (type: string), _col2 (type: int) 1 _col0 (type: string), _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 30 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 32 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 30 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -5680,13 +5680,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 30 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 28 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Reducer 3 Execution mode: llap @@ -5698,11 +5698,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col1 (type: int) outputColumnNames: _col1, _col3 - Statistics: Num rows: 59 Data size: 476 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 59 Data size: 476 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col3 (type: bigint) Reducer 4 Execution mode: llap @@ -5714,14 +5714,14 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col3, _col5 - Statistics: Num rows: 59 Data size: 948 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), _col3 (type: bigint), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 59 Data size: 948 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 59 Data size: 948 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -5761,10 +5761,10 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 32 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 29 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.96875 + minReductionHashAggr: 0.9655172 mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -5799,20 +5799,20 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(_col1) <= 1) (type: boolean) - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_dml.q.out b/ql/src/test/results/clientpositive/llap/tez_dml.q.out index d716b63012..bdc6ee6f7b 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -87,18 +87,14 @@ STAGE PLANS: aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/tez_union2.q.out b/ql/src/test/results/clientpositive/llap/tez_union2.q.out index 762a2a51d0..f809d5d11d 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union2.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union2.q.out @@ -66,13 +66,13 @@ STAGE PLANS: minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 4 @@ -128,10 +128,10 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -152,23 +152,23 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 54984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Union 2 Vertex: Union 2 Union 5 diff --git a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out index d5bc1790d4..8b7e705955 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out @@ -4084,7 +4084,7 @@ STAGE PLANS: keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 114750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), CAST( _col2 AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out index aa92f46f17..a290f37686 100644 --- a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out +++ b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out @@ -1159,7 +1159,7 @@ STAGE PLANS: keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 115750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 71000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), CAST( _col2 AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 @@ -2998,14 +2998,14 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 25000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 12500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 69000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 34500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 69000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 34500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -3014,7 +3014,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 250 Data size: 69000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 125 Data size: 34500 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.99 @@ -6876,13 +6876,13 @@ STAGE PLANS: minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 5 Map Operator Tree: @@ -6937,20 +6937,20 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 187 Data size: 34782 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 187 Data size: 34782 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 187 Data size: 34782 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -6959,7 +6959,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string) outputColumnNames: col1, col2, col3 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 187 Data size: 34782 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll'), compute_stats(col3, 'hll') minReductionHashAggr: 0.99 @@ -7002,23 +7002,23 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 54984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 408 Data size: 72624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: COMPLETE Union 2 Vertex: Union 2 Union 6 @@ -9333,13 +9333,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 9 Execution mode: vectorized Reduce Operator Tree: @@ -9995,14 +9998,14 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -10011,10 +10014,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: key, cnt - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(cnt, 'hll') - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.8 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out index 69c43706b2..2356da645f 100644 --- a/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out +++ b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out @@ -1152,10 +1152,10 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1389,10 +1389,10 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2232,10 +2232,10 @@ STAGE PLANS: keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 87 Data size: 16704 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 43 Data size: 8256 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 87 Data size: 16704 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 43 Data size: 8256 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_6.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_6.q.out index e899da5c1f..738ba89bb2 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_6.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_6.q.out @@ -742,18 +742,14 @@ STAGE PLANS: aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out index e74bc44680..9a2546d638 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out @@ -554,7 +554,7 @@ STAGE PLANS: keys: _col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 85 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 42 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 @@ -562,7 +562,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [1] - Statistics: Num rows: 85 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 42 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z @@ -571,7 +571,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 85 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 42 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -588,13 +588,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0] - Statistics: Num rows: 85 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 42 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 85 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 42 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1097,7 +1097,7 @@ STAGE PLANS: keys: _col1 (type: int), _col0 (type: int) mode: complete outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 500 Data size: 80000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), (UDFToDouble(_col2) / _col3) (type: double), _col4 (type: double), (_col4 / _col5) (type: double), _col6 (type: decimal(38,18)), (_col6 / _col7) (type: decimal(38,18)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 @@ -1106,7 +1106,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [0, 1, 2, 9, 4, 10, 6, 12] selectExpressions: DoubleColDivideLongColumn(col 8:double, col 3:bigint)(children: CastLongToDouble(col 2:bigint) -> 8:double) -> 9:double, DoubleColDivideLongColumn(col 4:double, col 5:bigint) -> 10:double, DecimalColDivideDecimalColumn(col 6:decimal(38,18), col 11:decimal(19,0))(children: CastLongToDecimal(col 7:bigint) -> 11:decimal(19,0)) -> 12:decimal(38,18) - Statistics: Num rows: 500 Data size: 132000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 66000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz @@ -1115,7 +1115,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 500 Data size: 132000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 66000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: decimal(38,18)), _col7 (type: decimal(38,18)) Reducer 3 Execution mode: vectorized, llap @@ -1133,13 +1133,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7] - Statistics: Num rows: 500 Data size: 132000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 66000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 500 Data size: 132000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 66000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out index cb086bd5a3..beb3eb503d 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out @@ -1529,37 +1529,40 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: decimal(15,2), _col1: decimal(15,2), _col2: double - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: sum_window_0 - arguments: _col2 - name: sum - window function: GenericUDAFSumDouble - window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: sum_window_0 (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)), _col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: decimal(15,2), _col1: decimal(15,2), _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: RANGE PRECEDING(MAX)~CURRENT + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: sum_window_0 (type: double) + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2879,37 +2882,40 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: decimal(7,2), _col1: decimal(7,2), _col2: double - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: sum_window_0 - arguments: _col2 - name: sum - window function: GenericUDAFSumDouble - window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: sum_window_0 (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: decimal(7,2), _col1: decimal(7,2), _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: RANGE PRECEDING(MAX)~CURRENT + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: sum_window_0 (type: double) + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out b/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out index d4d22d05d8..d806aad653 100644 --- a/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out @@ -142,37 +142,40 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: int, _col2: double, _col3: bigint - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col0 ASC NULLS FIRST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: sum_window_0 - arguments: _col1 - name: sum - window function: GenericUDAFSumLong - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: string), (_col2 / _col3) (type: double), sum_window_0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: _col0 (type: string), _col1 (type: int), _col2 (type: double), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int, _col2: double, _col3: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 ASC NULLS FIRST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col1 + name: sum + window function: GenericUDAFSumLong + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col1 (type: int), _col0 (type: string), (_col2 / _col3) (type: double), sum_window_0 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out index ca3c6337bf..e1a10916e5 100644 --- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out @@ -4879,36 +4879,39 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: double - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col0 ASC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: sum_window_0 - arguments: round(_col2, 2) - name: sum - window function: GenericUDAFSumDouble - window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), round(_col2, 2) (type: double), round(sum_window_0, 2) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 13 Data size: 2678 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 ASC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: round(_col2, 2) + name: sum + window function: GenericUDAFSumDouble + window frame: RANGE PRECEDING(MAX)~CURRENT + Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), round(_col2, 2) (type: double), round(sum_window_0, 2) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 13 Data size: 2678 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: double), _col3 (type: double) + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 13 Data size: 2678 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: double), _col3 (type: double) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out index 36276e1fc9..82233e0e79 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out @@ -627,22 +627,29 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 131 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Limit Vectorization: - className: VectorLimitOperator + Select Operator + expressions: _col0 (type: tinyint) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false + projectedOutputColumnNums: [0] + Limit + Number of rows: 20 + Limit Vectorization: + className: VectorLimitOperator + native: true Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out index 640e8f0dc3..500882866c 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out @@ -1413,52 +1413,55 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: int - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col1 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - window function definition - alias: dense_rank_window_1 - arguments: _col1 - name: dense_rank - window function: GenericUDAFDenseRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - window function definition - alias: lag_window_2 - arguments: _col2, 1, _col2 - name: lag - window function: GenericUDAFLagEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), _col2 (type: int), (_col2 - lag_window_2) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 13 Data size: 3107 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: dense_rank_window_1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: lag_window_2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), _col2 (type: int), (_col2 - lag_window_2) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 13 Data size: 3107 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 3107 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/query51.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/query51.q.out index 257cb58c50..c372e12603 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/query51.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/query51.q.out @@ -114,22 +114,22 @@ Stage-0 limit:100 Stage-1 Reducer 6 vectorized - File Output Operator [FS_117] - Limit [LIM_116] (rows=100 width=636) + File Output Operator [FS_119] + Limit [LIM_118] (rows=100 width=636) Number of rows:100 - Select Operator [SEL_115] (rows=363803676 width=636) + Select Operator [SEL_117] (rows=363803676 width=636) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Reducer 5 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_114] - Select Operator [SEL_113] (rows=363803676 width=636) + SHUFFLE [RS_116] + Select Operator [SEL_115] (rows=363803676 width=636) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Top N Key Operator [TNK_112] (rows=363803676 width=420) + Top N Key Operator [TNK_114] (rows=363803676 width=420) keys:CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END, CASE WHEN (_col4 is not null) THEN (_col4) ELSE (_col1) END,top n:100 - Filter Operator [FIL_111] (rows=363803676 width=420) + Filter Operator [FIL_113] (rows=363803676 width=420) predicate:(max_window_0 > max_window_1) - PTF Operator [PTF_110] (rows=1091411029 width=420) + PTF Operator [PTF_112] (rows=1091411029 width=420) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"CASE WHEN (_col4 is not null) THEN (_col4) ELSE (_col1) END ASC NULLS LAST","partition by:":"CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END"}] - Select Operator [SEL_109] (rows=1091411029 width=420) + Select Operator [SEL_111] (rows=1091411029 width=420) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_41] @@ -143,44 +143,46 @@ Stage-0 Output:["_col0","_col1","_col2"] PTF Operator [PTF_35] (rows=3442937 width=210) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS LAST","partition by:":"_col0"}] - Group By Operator [GBY_31] (rows=3442937 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=24992810 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_85] (rows=24992810 width=209) - Conds:RS_107._col0=RS_91._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 7 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_91] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=317 width=98) - Output:["_col0","_col1"] - Filter Operator [FIL_87] (rows=317 width=102) - predicate:d_month_seq BETWEEN 1212 AND 1223 - TableScan [TS_3] (rows=73049 width=102) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_date","d_month_seq"] - <-Map 12 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_107] - PartitionCols:_col0 - Select Operator [SEL_106] (rows=143966864 width=119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_105] (rows=143966864 width=119) - predicate:(ws_sold_date_sk is not null and ws_sold_date_sk BETWEEN DynamicValue(RS_26_date_dim_d_date_sk_min) AND DynamicValue(RS_26_date_dim_d_date_sk_max) and in_bloom_filter(ws_sold_date_sk, DynamicValue(RS_26_date_dim_d_date_sk_bloom_filter))) - TableScan [TS_19] (rows=144002668 width=119) - default@web_sales,web_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ws_sold_date_sk","ws_item_sk","ws_sales_price"] - <-Reducer 11 [BROADCAST_EDGE] vectorized - BROADCAST [RS_104] - Group By Operator [GBY_103] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized - SHUFFLE [RS_96] - Group By Operator [GBY_94] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] - Select Operator [SEL_92] (rows=317 width=4) - Output:["_col0"] - Please refer to the previous Select Operator [SEL_88] + Select Operator [SEL_83] + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_31] (rows=3442937 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 9 [SIMPLE_EDGE] + SHUFFLE [RS_30] + PartitionCols:_col0 + Group By Operator [GBY_29] (rows=24992810 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 + Merge Join Operator [MERGEJOIN_85] (rows=24992810 width=209) + Conds:RS_108._col0=RS_91._col0(Inner),Output:["_col1","_col2","_col4"] + <-Map 7 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_91] + PartitionCols:_col0 + Select Operator [SEL_88] (rows=317 width=98) + Output:["_col0","_col1"] + Filter Operator [FIL_87] (rows=317 width=102) + predicate:d_month_seq BETWEEN 1212 AND 1223 + TableScan [TS_3] (rows=73049 width=102) + default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_date","d_month_seq"] + <-Map 12 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_108] + PartitionCols:_col0 + Select Operator [SEL_107] (rows=143966864 width=119) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_106] (rows=143966864 width=119) + predicate:(ws_sold_date_sk is not null and ws_sold_date_sk BETWEEN DynamicValue(RS_26_date_dim_d_date_sk_min) AND DynamicValue(RS_26_date_dim_d_date_sk_max) and in_bloom_filter(ws_sold_date_sk, DynamicValue(RS_26_date_dim_d_date_sk_bloom_filter))) + TableScan [TS_19] (rows=144002668 width=119) + default@web_sales,web_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ws_sold_date_sk","ws_item_sk","ws_sales_price"] + <-Reducer 11 [BROADCAST_EDGE] vectorized + BROADCAST [RS_105] + Group By Operator [GBY_104] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] + <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized + SHUFFLE [RS_96] + Group By Operator [GBY_94] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] + Select Operator [SEL_92] (rows=317 width=4) + Output:["_col0"] + Please refer to the previous Select Operator [SEL_88] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col0, _col1 @@ -188,37 +190,39 @@ Stage-0 Output:["_col0","_col1","_col2"] PTF Operator [PTF_16] (rows=3442937 width=210) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS LAST","partition by:":"_col0"}] - Group By Operator [GBY_12] (rows=3442937 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=95493908 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_84] (rows=95493908 width=180) - Conds:RS_101._col0=RS_89._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 7 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_89] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_88] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_101] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=550076554 width=114) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_99] (rows=550076554 width=114) - predicate:(ss_sold_date_sk is not null and ss_sold_date_sk BETWEEN DynamicValue(RS_7_date_dim_d_date_sk_min) AND DynamicValue(RS_7_date_dim_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_date_dim_d_date_sk_bloom_filter))) - TableScan [TS_0] (rows=575995635 width=114) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_item_sk","ss_sales_price"] - <-Reducer 8 [BROADCAST_EDGE] vectorized - BROADCAST [RS_98] - Group By Operator [GBY_97] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized - SHUFFLE [RS_95] - Group By Operator [GBY_93] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] - Select Operator [SEL_90] (rows=317 width=4) - Output:["_col0"] - Please refer to the previous Select Operator [SEL_88] + Select Operator [SEL_82] + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_12] (rows=3442937 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_11] + PartitionCols:_col0 + Group By Operator [GBY_10] (rows=95493908 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 + Merge Join Operator [MERGEJOIN_84] (rows=95493908 width=180) + Conds:RS_101._col0=RS_89._col0(Inner),Output:["_col1","_col2","_col4"] + <-Map 7 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_89] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_88] + <-Map 1 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_101] + PartitionCols:_col0 + Select Operator [SEL_100] (rows=550076554 width=114) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_99] (rows=550076554 width=114) + predicate:(ss_sold_date_sk is not null and ss_sold_date_sk BETWEEN DynamicValue(RS_7_date_dim_d_date_sk_min) AND DynamicValue(RS_7_date_dim_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_date_dim_d_date_sk_bloom_filter))) + TableScan [TS_0] (rows=575995635 width=114) + default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_item_sk","ss_sales_price"] + <-Reducer 8 [BROADCAST_EDGE] vectorized + BROADCAST [RS_98] + Group By Operator [GBY_97] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] + <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized + SHUFFLE [RS_95] + Group By Operator [GBY_93] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] + Select Operator [SEL_90] (rows=317 width=4) + Output:["_col0"] + Please refer to the previous Select Operator [SEL_88] diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/query53.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/query53.q.out index 06726ba4a1..0ab927414b 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/query53.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/query53.q.out @@ -86,7 +86,7 @@ Stage-0 SHUFFLE [RS_27] Select Operator [SEL_24] (rows=25 width=228) Output:["_col0","_col1","_col2"] - Top N Key Operator [TNK_41] (rows=25 width=228) + Top N Key Operator [TNK_40] (rows=25 width=228) keys:avg_window_0, _col2, _col0,top n:100 Filter Operator [FIL_36] (rows=25 width=228) predicate:CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (false) END @@ -94,7 +94,7 @@ Stage-0 Output:["avg_window_0","_col0","_col2"] PTF Operator [PTF_22] (rows=50 width=116) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_19] (rows=50 width=116) + Select Operator [SEL_19] (rows=50 width=120) Output:["_col0","_col2"] Group By Operator [GBY_18] (rows=50 width=120) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/query63.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/query63.q.out index cdcd316388..1b84b74e5f 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/query63.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/query63.q.out @@ -88,7 +88,7 @@ Stage-0 SHUFFLE [RS_27] Select Operator [SEL_24] (rows=65 width=228) Output:["_col0","_col1","_col2"] - Top N Key Operator [TNK_41] (rows=65 width=228) + Top N Key Operator [TNK_40] (rows=65 width=228) keys:_col0, avg_window_0, _col2,top n:100 Filter Operator [FIL_36] (rows=65 width=228) predicate:CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (false) END @@ -96,7 +96,7 @@ Stage-0 Output:["avg_window_0","_col0","_col2"] PTF Operator [PTF_22] (rows=130 width=116) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_19] (rows=130 width=116) + Select Operator [SEL_19] (rows=130 width=120) Output:["_col0","_col2"] Group By Operator [GBY_18] (rows=130 width=120) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 diff --git a/ql/src/test/results/clientpositive/perf/tez/query51.q.out b/ql/src/test/results/clientpositive/perf/tez/query51.q.out index 8e3c53dfc9..74d8f8a8ce 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query51.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query51.q.out @@ -114,22 +114,22 @@ Stage-0 limit:100 Stage-1 Reducer 6 vectorized - File Output Operator [FS_117] - Limit [LIM_116] (rows=100 width=636) + File Output Operator [FS_119] + Limit [LIM_118] (rows=100 width=636) Number of rows:100 - Select Operator [SEL_115] (rows=363803676 width=636) + Select Operator [SEL_117] (rows=363803676 width=636) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Reducer 5 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_114] - Select Operator [SEL_113] (rows=363803676 width=636) + SHUFFLE [RS_116] + Select Operator [SEL_115] (rows=363803676 width=636) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Top N Key Operator [TNK_112] (rows=363803676 width=420) + Top N Key Operator [TNK_114] (rows=363803676 width=420) keys:CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END, CASE WHEN (_col4 is not null) THEN (_col4) ELSE (_col1) END,top n:100 - Filter Operator [FIL_111] (rows=363803676 width=420) + Filter Operator [FIL_113] (rows=363803676 width=420) predicate:(max_window_0 > max_window_1) - PTF Operator [PTF_110] (rows=1091411029 width=420) + PTF Operator [PTF_112] (rows=1091411029 width=420) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"CASE WHEN (_col4 is not null) THEN (_col4) ELSE (_col1) END ASC NULLS LAST","partition by:":"CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END"}] - Select Operator [SEL_109] (rows=1091411029 width=420) + Select Operator [SEL_111] (rows=1091411029 width=420) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_41] @@ -143,44 +143,46 @@ Stage-0 Output:["_col0","_col1","_col2"] PTF Operator [PTF_35] (rows=3442937 width=210) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS LAST","partition by:":"_col0"}] - Group By Operator [GBY_31] (rows=3442937 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=24992810 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_85] (rows=24992810 width=209) - Conds:RS_107._col0=RS_91._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 7 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_91] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=317 width=98) - Output:["_col0","_col1"] - Filter Operator [FIL_87] (rows=317 width=102) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=102) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_date","d_month_seq"] - <-Map 12 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_107] - PartitionCols:_col0 - Select Operator [SEL_106] (rows=143966864 width=119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_105] (rows=143966864 width=119) - predicate:(ws_sold_date_sk is not null and ws_item_sk is not null and ws_sold_date_sk BETWEEN DynamicValue(RS_26_date_dim_d_date_sk_min) AND DynamicValue(RS_26_date_dim_d_date_sk_max) and in_bloom_filter(ws_sold_date_sk, DynamicValue(RS_26_date_dim_d_date_sk_bloom_filter))) - TableScan [TS_19] (rows=144002668 width=119) - default@web_sales,web_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ws_sold_date_sk","ws_item_sk","ws_sales_price"] - <-Reducer 11 [BROADCAST_EDGE] vectorized - BROADCAST [RS_104] - Group By Operator [GBY_103] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized - SHUFFLE [RS_96] - Group By Operator [GBY_94] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] - Select Operator [SEL_92] (rows=317 width=4) - Output:["_col0"] - Please refer to the previous Select Operator [SEL_88] + Select Operator [SEL_83] + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_31] (rows=3442937 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 9 [SIMPLE_EDGE] + SHUFFLE [RS_30] + PartitionCols:_col0 + Group By Operator [GBY_29] (rows=24992810 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 + Merge Join Operator [MERGEJOIN_85] (rows=24992810 width=209) + Conds:RS_108._col0=RS_91._col0(Inner),Output:["_col1","_col2","_col4"] + <-Map 7 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_91] + PartitionCols:_col0 + Select Operator [SEL_88] (rows=317 width=98) + Output:["_col0","_col1"] + Filter Operator [FIL_87] (rows=317 width=102) + predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=102) + default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_date","d_month_seq"] + <-Map 12 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_108] + PartitionCols:_col0 + Select Operator [SEL_107] (rows=143966864 width=119) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_106] (rows=143966864 width=119) + predicate:(ws_sold_date_sk is not null and ws_item_sk is not null and ws_sold_date_sk BETWEEN DynamicValue(RS_26_date_dim_d_date_sk_min) AND DynamicValue(RS_26_date_dim_d_date_sk_max) and in_bloom_filter(ws_sold_date_sk, DynamicValue(RS_26_date_dim_d_date_sk_bloom_filter))) + TableScan [TS_19] (rows=144002668 width=119) + default@web_sales,web_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ws_sold_date_sk","ws_item_sk","ws_sales_price"] + <-Reducer 11 [BROADCAST_EDGE] vectorized + BROADCAST [RS_105] + Group By Operator [GBY_104] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] + <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized + SHUFFLE [RS_96] + Group By Operator [GBY_94] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] + Select Operator [SEL_92] (rows=317 width=4) + Output:["_col0"] + Please refer to the previous Select Operator [SEL_88] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col0, _col1 @@ -188,37 +190,39 @@ Stage-0 Output:["_col0","_col1","_col2"] PTF Operator [PTF_16] (rows=3442937 width=210) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS LAST","partition by:":"_col0"}] - Group By Operator [GBY_12] (rows=3442937 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=95493908 width=210) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_84] (rows=95493908 width=180) - Conds:RS_101._col0=RS_89._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 7 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_89] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_88] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_101] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=550076554 width=114) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_99] (rows=550076554 width=114) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_sold_date_sk BETWEEN DynamicValue(RS_7_date_dim_d_date_sk_min) AND DynamicValue(RS_7_date_dim_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_date_dim_d_date_sk_bloom_filter))) - TableScan [TS_0] (rows=575995635 width=114) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_item_sk","ss_sales_price"] - <-Reducer 8 [BROADCAST_EDGE] vectorized - BROADCAST [RS_98] - Group By Operator [GBY_97] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized - SHUFFLE [RS_95] - Group By Operator [GBY_93] (rows=1 width=12) - Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] - Select Operator [SEL_90] (rows=317 width=4) - Output:["_col0"] - Please refer to the previous Select Operator [SEL_88] + Select Operator [SEL_82] + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_12] (rows=3442937 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_11] + PartitionCols:_col0 + Group By Operator [GBY_10] (rows=95493908 width=210) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 + Merge Join Operator [MERGEJOIN_84] (rows=95493908 width=180) + Conds:RS_101._col0=RS_89._col0(Inner),Output:["_col1","_col2","_col4"] + <-Map 7 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_89] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_88] + <-Map 1 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_101] + PartitionCols:_col0 + Select Operator [SEL_100] (rows=550076554 width=114) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_99] (rows=550076554 width=114) + predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_sold_date_sk BETWEEN DynamicValue(RS_7_date_dim_d_date_sk_min) AND DynamicValue(RS_7_date_dim_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_date_dim_d_date_sk_bloom_filter))) + TableScan [TS_0] (rows=575995635 width=114) + default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_item_sk","ss_sales_price"] + <-Reducer 8 [BROADCAST_EDGE] vectorized + BROADCAST [RS_98] + Group By Operator [GBY_97] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] + <-Map 7 [CUSTOM_SIMPLE_EDGE] vectorized + SHUFFLE [RS_95] + Group By Operator [GBY_93] (rows=1 width=12) + Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] + Select Operator [SEL_90] (rows=317 width=4) + Output:["_col0"] + Please refer to the previous Select Operator [SEL_88] diff --git a/ql/src/test/results/clientpositive/perf/tez/query53.q.out b/ql/src/test/results/clientpositive/perf/tez/query53.q.out index 2d2c0c374e..c3f5a60cff 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query53.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query53.q.out @@ -87,7 +87,7 @@ Stage-0 SHUFFLE [RS_33] Select Operator [SEL_30] (rows=25 width=228) Output:["_col0","_col1","_col2"] - Top N Key Operator [TNK_52] (rows=25 width=228) + Top N Key Operator [TNK_51] (rows=25 width=228) keys:avg_window_0, _col2, _col0,top n:100 Filter Operator [FIL_46] (rows=25 width=228) predicate:CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (false) END @@ -95,7 +95,7 @@ Stage-0 Output:["avg_window_0","_col0","_col2"] PTF Operator [PTF_28] (rows=50 width=116) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_25] (rows=50 width=116) + Select Operator [SEL_25] (rows=50 width=120) Output:["_col0","_col2"] Group By Operator [GBY_24] (rows=50 width=120) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 diff --git a/ql/src/test/results/clientpositive/perf/tez/query63.q.out b/ql/src/test/results/clientpositive/perf/tez/query63.q.out index 6d7a54a808..8f9dc621ac 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query63.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query63.q.out @@ -89,7 +89,7 @@ Stage-0 SHUFFLE [RS_33] Select Operator [SEL_30] (rows=65 width=228) Output:["_col0","_col1","_col2"] - Top N Key Operator [TNK_52] (rows=65 width=228) + Top N Key Operator [TNK_51] (rows=65 width=228) keys:_col0, avg_window_0, _col2,top n:100 Filter Operator [FIL_46] (rows=65 width=228) predicate:CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (false) END @@ -97,7 +97,7 @@ Stage-0 Output:["avg_window_0","_col0","_col2"] PTF Operator [PTF_28] (rows=130 width=116) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_25] (rows=130 width=116) + Select Operator [SEL_25] (rows=130 width=120) Output:["_col0","_col2"] Group By Operator [GBY_24] (rows=130 width=120) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 -- 2.17.2 (Apple Git-113)