diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index eca5a8b323..1dde26177f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -492,6 +492,9 @@ private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext pars SelectDesc select = new SelectDesc(keyExprs, outputNames); SelectOperator selectOp = (SelectOperator) OperatorFactory.getAndMakeChild(select, new RowSchema(selectColInfos), parentOfRS); + Map selectColumnExprMap = new HashMap<>(); + selectColumnExprMap.put(outputNames.get(0), key); + selectOp.setColumnExprMap(selectColumnExprMap); // do a group by on the list to dedup float groupByMemoryUsage = diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 0104d1955e..07f0fcdc90 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -1300,7 +1300,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 @@ -1316,18 +1316,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (string) Target Input: srcpart Partition key expr: hr - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: llap LLAP IO: no inputs @@ -2916,7 +2916,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 @@ -2932,18 +2932,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (string) Target Input: srcpart Partition key expr: hr - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: llap LLAP IO: no inputs @@ -5126,7 +5126,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 @@ -5142,18 +5142,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (string) Target Input: srcpart Partition key expr: hr - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: llap LLAP IO: no inputs @@ -6813,7 +6813,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.5 @@ -6829,18 +6829,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: double) minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: llap LLAP IO: no inputs diff --git a/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out b/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out index 694ef329ac..3c944483fb 100644 --- a/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out @@ -276,7 +276,7 @@ Stage-0 Dynamic Partitioning Event Operator [EVENT_132] (rows=1 width=4) Group By Operator [GBY_131] (rows=1 width=4) Output:["_col0"],keys:_col0 - Select Operator [SEL_130] (rows=28 width=8) + Select Operator [SEL_130] (rows=28 width=4) Output:["_col0"] Please refer to the previous Map Join Operator [MAPJOIN_128] <-Map Join Operator [MAPJOIN_135] (rows=370371 width=4) diff --git a/ql/src/test/results/clientpositive/llap/groupby_rollup_empty2.q.out b/ql/src/test/results/clientpositive/llap/groupby_rollup_empty2.q.out index 0b7f84451d..900b16f53b 100644 --- a/ql/src/test/results/clientpositive/llap/groupby_rollup_empty2.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby_rollup_empty2.q.out @@ -113,7 +113,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.6666666 diff --git a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out index 96f2d88482..012569950c 100644 --- a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out @@ -2012,10 +2012,10 @@ STAGE PLANS: vectorProcessingMode: HASH projectedOutputColumnNums: [] keys: _col0 (type: tinyint) - minReductionHashAggr: 0.9 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ctinyint (tinyint) App Master Event Vectorization: @@ -2023,7 +2023,7 @@ STAGE PLANS: native: true Target Input: oft Partition key expr: ctinyint - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: all inputs diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_no_join_opt_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_no_join_opt_2.q.out index c5285ab40f..0796cd3174 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_no_join_opt_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_no_join_opt_2.q.out @@ -223,7 +223,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.6666666 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_1.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_1.q.out index cf33791549..3f19853e67 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_1.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_1.q.out @@ -223,7 +223,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.6666666 diff --git a/ql/src/test/results/clientpositive/llap/temp_table_llap_partitioned.q.out b/ql/src/test/results/clientpositive/llap/temp_table_llap_partitioned.q.out index 045e9eea3b..42a4e94855 100644 --- a/ql/src/test/results/clientpositive/llap/temp_table_llap_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/temp_table_llap_partitioned.q.out @@ -2012,10 +2012,10 @@ STAGE PLANS: vectorProcessingMode: HASH projectedOutputColumnNums: [] keys: _col0 (type: tinyint) - minReductionHashAggr: 0.9 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ctinyint (tinyint) App Master Event Vectorization: @@ -2023,7 +2023,7 @@ STAGE PLANS: native: true Target Input: oft Partition key expr: ctinyint - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: all inputs diff --git a/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out b/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out index 7440b76d08..393fdc3fda 100644 --- a/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out @@ -2259,18 +2259,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.697 + minReductionHashAggr: 0.691 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 303 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: v (string) Target Input: t2 Partition key expr: v - Statistics: Num rows: 303 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 4 Execution mode: vectorized, llap LLAP IO: no inputs diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 0e37c7f673..d7f8f73b9b 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -1198,7 +1198,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 @@ -1214,18 +1214,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (string) Target Input: srcpart Partition key expr: hr - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: all inputs @@ -3152,7 +3152,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 @@ -3168,18 +3168,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (string) Target Input: srcpart Partition key expr: hr - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: all inputs @@ -5662,7 +5662,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 @@ -5678,18 +5678,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.0 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (string) Target Input: srcpart Partition key expr: hr - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: all inputs @@ -7685,7 +7685,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.5 @@ -7701,18 +7701,18 @@ STAGE PLANS: Select Operator expressions: _col1 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: double) minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: hr (int) Target Input: srcpart_orc_n0 Partition key expr: UDFToDouble(hr) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Execution mode: llap LLAP IO: all inputs diff --git a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out index 630e2d0181..7ecb0fc306 100644 --- a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out @@ -151,18 +151,18 @@ STAGE PLANS: Select Operator expressions: _col2 (type: char(50)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 474 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: char(50)) minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 134 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: gpa (char(5)) Target Input: c2 Partition key expr: gpa - Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 134 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 3 Execution mode: vectorized Map Vectorization: