diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java index 055da42e10..f3930a118c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java @@ -24,20 +24,28 @@ import java.util.Map; import java.util.Set; +import com.google.common.collect.ImmutableList; import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexPermuteInputsShuttle; import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql2rel.CorrelationReferenceFinder; import org.apache.calcite.sql2rel.RelFieldTrimmer; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; @@ -83,6 +91,40 @@ public HiveRelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder, boolea this.fetchStats = fetchStats; } + /** + * Trims the fields of an input relational expression. + * + * @param rel Relational expression + * @param input Input relational expression, whose fields to trim + * @param fieldsUsed Bitmap of fields needed by the consumer + * @return New relational expression and its field mapping + */ + protected TrimResult trimChild( + RelNode rel, + RelNode input, + final ImmutableBitSet fieldsUsed, + Set extraFields) { + final ImmutableBitSet.Builder fieldsUsedBuilder = fieldsUsed.rebuild(); + + // Correlating variables are a means for other relational expressions to use + // fields. + for (final CorrelationId correlation : rel.getVariablesSet()) { + rel.accept( + new CorrelationReferenceFinder() { + protected RexNode handle(RexFieldAccess fieldAccess) { + final RexCorrelVariable v = + (RexCorrelVariable) fieldAccess.getReferenceExpr(); + if (v.id.equals(correlation)) { + fieldsUsedBuilder.set(fieldAccess.getField().getIndex()); + } + return fieldAccess; + } + }); + } + + return dispatchTrimFields(input, fieldsUsedBuilder.build(), extraFields); + } + /** * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin}. diff --git a/ql/src/test/results/clientpositive/bucket_map_join_spark4.q.out b/ql/src/test/results/clientpositive/bucket_map_join_spark4.q.out index 688fdfa125..2cdf28e399 100644 --- a/ql/src/test/results/clientpositive/bucket_map_join_spark4.q.out +++ b/ql/src/test/results/clientpositive/bucket_map_join_spark4.q.out @@ -111,12 +111,12 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col1 (type: string) - 1 _col1 (type: string) + 1 _col0 (type: string) Position of Big Table: 0 Stage: Stage-5 @@ -148,12 +148,12 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col3, _col5 + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Position of Big Table: 0 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col5 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -467,12 +467,12 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col1 (type: string) - 1 _col1 (type: string) + 1 _col0 (type: string) Position of Big Table: 0 Stage: Stage-5 @@ -504,12 +504,12 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col3, _col5 + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Position of Big Table: 0 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col5 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/complex_alias.q.out b/ql/src/test/results/clientpositive/complex_alias.q.out index 64e1f3787c..c690e8f193 100644 --- a/ql/src/test/results/clientpositive/complex_alias.q.out +++ b/ql/src/test/results/clientpositive/complex_alias.q.out @@ -115,7 +115,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -123,10 +123,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 1 Data size: 35 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col3 (type: string) + expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 35 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -192,7 +192,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col0 (type: string) - outputColumnNames: _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/druid_basic2.q.out b/ql/src/test/results/clientpositive/druid_basic2.q.out index 64e494d71a..08b64d6d82 100644 --- a/ql/src/test/results/clientpositive/druid_basic2.q.out +++ b/ql/src/test/results/clientpositive/druid_basic2.q.out @@ -545,10 +545,10 @@ STAGE PLANS: GatherStats: false Select Operator expressions: robot (type: string), floor_day (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float), UDFToInteger(robot) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int), _col2 (type: float) + key expressions: _col4 (type: int), _col2 (type: float) null sort order: az sort order: +- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out index 15f6d250cc..2c7decd16f 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out @@ -424,11 +424,11 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 1724 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE tag: 1 auto parallelism: false Execution mode: llap @@ -498,7 +498,7 @@ STAGE PLANS: 0 1 Position of Big Table: 0 - Statistics: Num rows: 1392 Data size: 238866 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1392 Data size: 233298 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out index e2b60025b5..26024b0c35 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out @@ -1622,7 +1622,7 @@ STAGE PLANS: Stage: Stage-0 Fetch Operator - limit: -1 + limit: 10 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out index cdca7c6a54..8a6b70daec 100644 --- a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out @@ -672,7 +672,7 @@ Stage-0 Reducer 2 llap File Output Operator [FS_16] Merge Join Operator [MERGEJOIN_27] (rows=604/1166 width=95) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col1 @@ -692,9 +692,9 @@ Stage-0 default@tab,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] - PartitionCols:_col1 + PartitionCols:_col0 Select Operator [SEL_8] (rows=242/242 width=91) - Output:["_col1"] + Output:["_col0"] Filter Operator [FIL_24] (rows=242/242 width=91) predicate:value is not null TableScan [TS_6] (rows=242/242 width=91) @@ -775,7 +775,7 @@ Stage-0 Reducer 2 llap File Output Operator [FS_16] Merge Join Operator [MERGEJOIN_27] (rows=604/1166 width=95) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col1 @@ -795,9 +795,9 @@ Stage-0 default@tab,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] - PartitionCols:_col1 + PartitionCols:_col0 Select Operator [SEL_8] (rows=242/242 width=91) - Output:["_col1"] + Output:["_col0"] Filter Operator [FIL_24] (rows=242/242 width=91) predicate:value is not null TableScan [TS_6] (rows=242/242 width=91) @@ -967,7 +967,7 @@ Stage-0 Reduce Output Operator [RS_24] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_49] (rows=604/1166 width=4) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0"] + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col1 @@ -987,9 +987,9 @@ Stage-0 default@tab,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_13] - PartitionCols:_col1 + PartitionCols:_col0 Select Operator [SEL_8] (rows=242/242 width=91) - Output:["_col1"] + Output:["_col0"] Filter Operator [FIL_44] (rows=242/242 width=91) predicate:value is not null TableScan [TS_6] (rows=242/242 width=91) diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 8ecab56b26..43ca0d9ad7 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -1210,10 +1210,10 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Stage-0 Fetch Operator - limit:-1 + limit:1 Stage-1 Reducer 3 llap - File Output Operator [FS_11] + File Output Operator [FS_10] Limit [LIM_9] (rows=1 width=97) Number of rows:1 Select Operator [SEL_8] (rows=10 width=97) @@ -1395,7 +1395,7 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] llap SHUFFLE [RS_8] Select Operator [SEL_6] (rows=3 width=105) - Output:["_col0","_col1","_col2","_col5"] + Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_5] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 <-Map 1 [SIMPLE_EDGE] llap diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 502fcfea8e..819e5136d5 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -1728,7 +1728,7 @@ Stage-0 Reducer 2 llap File Output Operator [FS_16] Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col1 @@ -1748,9 +1748,9 @@ Stage-0 default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] - PartitionCols:_col1 + PartitionCols:_col0 Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] + Output:["_col0"] Filter Operator [FIL_24] (rows=242 width=10) predicate:value is not null TableScan [TS_6] (rows=242 width=10) @@ -1803,7 +1803,7 @@ Stage-0 Reducer 2 llap File Output Operator [FS_16] Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col1 @@ -1823,9 +1823,9 @@ Stage-0 default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] - PartitionCols:_col1 + PartitionCols:_col0 Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] + Output:["_col0"] Filter Operator [FIL_24] (rows=242 width=10) predicate:value is not null TableScan [TS_6] (rows=242 width=10) @@ -1959,7 +1959,7 @@ Stage-0 Reduce Output Operator [RS_24] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_50] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0"] + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col1 @@ -1979,9 +1979,9 @@ Stage-0 default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_13] - PartitionCols:_col1 + PartitionCols:_col0 Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] + Output:["_col0"] Filter Operator [FIL_45] (rows=242 width=10) predicate:value is not null TableScan [TS_6] (rows=242 width=10) diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index b067e45e2e..72d3d398c2 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -1331,11 +1331,11 @@ STAGE PLANS: Select Operator expressions: _col1 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: double) sort order: + - Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 Reducer 3 Execution mode: llap @@ -1357,7 +1357,7 @@ STAGE PLANS: Stage: Stage-0 Fetch Operator - limit: -1 + limit: 100 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out index 470cffe7cc..051b2dbe1f 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out @@ -1283,11 +1283,11 @@ STAGE PLANS: Select Operator expressions: _col1 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: double) sort order: + - Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 Reducer 3 Execution mode: llap @@ -1309,7 +1309,7 @@ STAGE PLANS: Stage: Stage-0 Fetch Operator - limit: -1 + limit: 100 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/mergejoin.q.out b/ql/src/test/results/clientpositive/llap/mergejoin.q.out index c07afdf52d..2661c434c1 100644 --- a/ql/src/test/results/clientpositive/llap/mergejoin.q.out +++ b/ql/src/test/results/clientpositive/llap/mergejoin.q.out @@ -2596,16 +2596,16 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 22022 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true projectedOutputColumnNums: [1] Statistics: Num rows: 242 Data size: 22022 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumnNums: [1] @@ -2649,16 +2649,16 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true projectedOutputColumnNums: [1] Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumnNums: [1] @@ -2691,8 +2691,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 391 Data size: 3128 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -3129,16 +3129,16 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 22022 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true projectedOutputColumnNums: [1] Statistics: Num rows: 242 Data size: 22022 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumnNums: [1] @@ -3182,16 +3182,16 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true projectedOutputColumnNums: [1] Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumnNums: [1] @@ -3224,8 +3224,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 391 Data size: 3128 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index b6bc569f19..ec3c286102 100644 --- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -1166,11 +1166,11 @@ STAGE PLANS: Select Operator expressions: _col1 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: double) sort order: + - Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 Reducer 3 Execution mode: llap @@ -1193,7 +1193,7 @@ STAGE PLANS: Stage: Stage-0 Fetch Operator - limit: -1 + limit: 70 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 90a9048430..e894a44451 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -6129,11 +6129,11 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - 2 _col3 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4, _col7 + 2 _col2 (type: int) + outputColumnNames: _col0, _col1, _col3, _col4, _col6 Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col6 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int) @@ -6171,14 +6171,14 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -6188,14 +6188,14 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col3 (type: int) + key expressions: _col2 (type: int) sort order: + - Map-reduce partition columns: _col3 (type: int) + Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) Reducer 8 Execution mode: llap Reduce Operator Tree: @@ -6353,11 +6353,11 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - 2 _col3 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4, _col7 + 2 _col2 (type: int) + outputColumnNames: _col0, _col1, _col3, _col4, _col6 Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col6 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) @@ -6395,14 +6395,14 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Reducer 5 Execution mode: llap Reduce Operator Tree: @@ -6412,14 +6412,14 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col3 (type: int) + key expressions: _col2 (type: int) sort order: + - Map-reduce partition columns: _col3 (type: int) + Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) Reducer 7 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out index 3ba7d20332..01a86d1300 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -230,11 +230,11 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - 2 _col4 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5, _col9 + 2 _col3 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5, _col8 Statistics: Num rows: 87 Data size: 17226 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col9 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 43 Data size: 8514 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string) @@ -289,14 +289,14 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string), _col3 (type: boolean) + value expressions: _col1 (type: string), _col2 (type: boolean) Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -306,20 +306,20 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col2, _col3, _col4 + outputColumnNames: _col1, _col2, _col3 Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: string), _col2 (type: string) + key expressions: _col3 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col4 (type: string), _col2 (type: string) + Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: boolean) + value expressions: _col2 (type: boolean) Reduce Output Operator - key expressions: _col4 (type: string), _col2 (type: string) + key expressions: _col3 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col4 (type: string), _col2 (type: string) + Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: boolean) + value expressions: _col2 (type: boolean) Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -330,11 +330,11 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - 2 _col4 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col4, _col5, _col9 + 2 _col3 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col4, _col5, _col8 Statistics: Num rows: 87 Data size: 9309 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col9 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 43 Data size: 4601 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out b/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out index 3de862e302..705bcc3fad 100644 --- a/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out @@ -263,33 +263,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 242 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 230 Data size: 3316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 230 Data size: 3316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 230 Data size: 3316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 2 Map Operator Tree: TableScan alias: b - Statistics: Num rows: 500 Data size: 7216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 475 Data size: 6855 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 475 Data size: 6855 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -298,15 +298,15 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 0 Map 1 - Statistics: Num rows: 522 Data size: 7540 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 391 Data size: 3128 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -317,10 +317,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -374,33 +374,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 242 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 230 Data size: 3316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 230 Data size: 3316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 230 Data size: 3316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 2 Map Operator Tree: TableScan alias: b - Statistics: Num rows: 500 Data size: 7216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 475 Data size: 6855 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 475 Data size: 6855 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -409,15 +409,15 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 0 Map 1 - Statistics: Num rows: 522 Data size: 7540 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 391 Data size: 3128 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -428,10 +428,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -867,12 +867,12 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -886,12 +886,12 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -902,8 +902,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -963,12 +963,12 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -982,12 +982,12 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -998,8 +998,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1219,12 +1219,12 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1238,14 +1238,14 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) input vertices: 0 Map 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -1309,12 +1309,12 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1328,14 +1328,14 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) input vertices: 0 Map 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -1674,14 +1674,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s1 - Statistics: Num rows: 242 Data size: 924 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 230 Data size: 878 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 230 Data size: 878 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -1691,7 +1691,7 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 4 - Statistics: Num rows: 253 Data size: 965 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 382 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -1700,15 +1700,15 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 6 - Statistics: Num rows: 531 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1009 Data size: 8072 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -1716,33 +1716,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s3 - Statistics: Num rows: 242 Data size: 924 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 230 Data size: 878 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 230 Data size: 878 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 230 Data size: 878 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 5 Map Operator Tree: TableScan alias: s2 - Statistics: Num rows: 242 Data size: 924 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 230 Data size: 878 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 230 Data size: 878 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -1751,15 +1751,15 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 6 - Statistics: Num rows: 531 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1009 Data size: 8072 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -1767,24 +1767,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 500 Data size: 1904 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 475 Data size: 1808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 475 Data size: 1808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 475 Data size: 1808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 475 Data size: 1808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 3 @@ -1794,10 +1794,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1838,7 +1838,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1846,65 +1847,74 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - Statistics: Num rows: 242 Data size: 45994 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 22990 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 230 Data size: 43713 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 22990 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 230 Data size: 43713 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 22990 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 230 Data size: 43713 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 22990 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 3 Map Operator Tree: TableScan alias: t2 - Statistics: Num rows: 500 Data size: 94800 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 475 Data size: 90060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 475 Data size: 90060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 475 Data size: 90060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 4 + Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 230 Data size: 43713 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 475 Data size: 90060 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 522 Data size: 99066 Basic stats: COMPLETE Column stats: NONE + input vertices: + 0 Reducer 2 + Statistics: Num rows: 391 Data size: 3128 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 5 Execution mode: llap @@ -1913,10 +1923,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out index dee20d1aad..f7f3014a7f 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out @@ -803,7 +803,7 @@ STAGE PLANS: Stage: Stage-0 Fetch Operator - limit: -1 + limit: 10 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out index 0da9a4560a..c21a4fc091 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out @@ -539,7 +539,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [1] - Statistics: Num rows: 82 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 82 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -547,7 +547,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 82 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 82 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: diff --git a/ql/src/test/results/clientpositive/mergejoin.q.out b/ql/src/test/results/clientpositive/mergejoin.q.out index 5285e456e2..41997db970 100644 --- a/ql/src/test/results/clientpositive/mergejoin.q.out +++ b/ql/src/test/results/clientpositive/mergejoin.q.out @@ -2025,12 +2025,12 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE TableScan alias: b @@ -2041,12 +2041,12 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE Map Vectorization: enabled: false @@ -2060,8 +2060,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 550 Data size: 51700 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2383,12 +2383,12 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE TableScan alias: b @@ -2399,12 +2399,12 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE Map Vectorization: enabled: false @@ -2418,8 +2418,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 550 Data size: 51700 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() diff --git a/ql/src/test/results/clientpositive/order3.q.out b/ql/src/test/results/clientpositive/order3.q.out index bff29244f6..e19698c5e8 100644 --- a/ql/src/test/results/clientpositive/order3.q.out +++ b/ql/src/test/results/clientpositive/order3.q.out @@ -270,27 +270,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: bigint) - outputColumnNames: _col0, _col1 + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: bigint), 'AAA' (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 7 Data size: 70 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), 'AAA' (type: string) - outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator - limit: -1 + limit: 3 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/perf/spark/query11.q.out b/ql/src/test/results/clientpositive/perf/spark/query11.q.out index f8739ad405..e46aa2106b 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query11.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query11.q.out @@ -441,14 +441,14 @@ STAGE PLANS: Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: decimal(18,2)) - outputColumnNames: _col0, _col6 + outputColumnNames: _col0, _col1 Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE - value expressions: _col6 (type: decimal(18,2)) + value expressions: _col1 (type: decimal(18,2)) Reducer 16 Reduce Operator Tree: Join Operator @@ -501,14 +501,14 @@ STAGE PLANS: Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col7 (type: decimal(18,2)) - outputColumnNames: _col0, _col3, _col7 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: string), _col7 (type: decimal(18,2)) + value expressions: _col1 (type: string), _col2 (type: decimal(18,2)) Reducer 2 Reduce Operator Tree: Join Operator @@ -582,12 +582,16 @@ STAGE PLANS: Filter Operator predicate: (_col7 > 0) (type: boolean) Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col7 (type: decimal(18,2)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE - value expressions: _col7 (type: decimal(18,2)) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(18,2)) Reducer 3 Reduce Operator Tree: Join Operator @@ -624,14 +628,14 @@ STAGE PLANS: Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: decimal(18,2)) - outputColumnNames: _col0, _col7 + outputColumnNames: _col0, _col1 Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE - value expressions: _col7 (type: decimal(18,2)) + value expressions: _col1 (type: decimal(18,2)) Reducer 5 Reduce Operator Tree: Join Operator @@ -644,13 +648,13 @@ STAGE PLANS: 1 _col0 (type: string) 2 _col0 (type: string) 3 _col0 (type: string) - outputColumnNames: _col7, _col14, _col18, _col22, _col30 + outputColumnNames: _col1, _col3, _col5, _col6, _col8 Statistics: Num rows: 1149975359 Data size: 101451160012 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col7 / _col30) > (_col22 / _col14)) (type: boolean) + predicate: ((_col1 / _col8) > (_col6 / _col3)) (type: boolean) Statistics: Num rows: 383325119 Data size: 33817053278 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col18 (type: string) + expressions: _col5 (type: string) outputColumnNames: _col0 Statistics: Num rows: 383325119 Data size: 33817053278 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator diff --git a/ql/src/test/results/clientpositive/perf/spark/query4.q.out b/ql/src/test/results/clientpositive/perf/spark/query4.q.out index 5933d0e815..75c4fbb4cf 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query4.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query4.q.out @@ -619,12 +619,16 @@ STAGE PLANS: Filter Operator predicate: (_col7 > 0) (type: boolean) Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col7 (type: decimal(24,6)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE - value expressions: _col7 (type: decimal(24,6)) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(24,6)) Reducer 16 Reduce Operator Tree: Join Operator @@ -677,14 +681,14 @@ STAGE PLANS: Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col7 (type: decimal(24,6)) - outputColumnNames: _col0, _col3, _col7 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: string), _col7 (type: decimal(24,6)) + value expressions: _col1 (type: string), _col2 (type: decimal(24,6)) Reducer 2 Reduce Operator Tree: Join Operator @@ -758,12 +762,16 @@ STAGE PLANS: Filter Operator predicate: (_col7 > 0) (type: boolean) Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col7 (type: decimal(24,6)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE - value expressions: _col7 (type: decimal(24,6)) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(24,6)) Reducer 28 Reduce Operator Tree: Join Operator @@ -842,14 +850,14 @@ STAGE PLANS: Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: decimal(24,6)) - outputColumnNames: _col0, _col7 + outputColumnNames: _col0, _col1 Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE - value expressions: _col7 (type: decimal(24,6)) + value expressions: _col1 (type: decimal(24,6)) Reducer 34 Reduce Operator Tree: Join Operator @@ -907,12 +915,16 @@ STAGE PLANS: Filter Operator predicate: (_col7 > 0) (type: boolean) Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col7 (type: decimal(24,6)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE - value expressions: _col7 (type: decimal(24,6)) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(24,6)) Reducer 4 Reduce Operator Tree: Group By Operator @@ -923,14 +935,14 @@ STAGE PLANS: Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: decimal(24,6)) - outputColumnNames: _col0, _col7 + outputColumnNames: _col0, _col1 Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE - value expressions: _col7 (type: decimal(24,6)) + value expressions: _col1 (type: decimal(24,6)) Reducer 5 Reduce Operator Tree: Join Operator @@ -947,13 +959,13 @@ STAGE PLANS: 3 _col0 (type: string) 4 _col0 (type: string) 5 _col0 (type: string) - outputColumnNames: _col7, _col15, _col19, _col23, _col31, _col39, _col47 + outputColumnNames: _col1, _col3, _col5, _col6, _col8, _col10, _col12 Statistics: Num rows: 1916625598 Data size: 169085266687 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 / _col31) > (_col23 / _col15)) and ((_col7 / _col31) > (_col39 / _col47))) (type: boolean) + predicate: (((_col1 / _col8) > (_col10 / _col12)) and ((_col1 / _col8) > (_col6 / _col3))) (type: boolean) Statistics: Num rows: 212958399 Data size: 18787251785 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col19 (type: string) + expressions: _col5 (type: string) outputColumnNames: _col0 Statistics: Num rows: 212958399 Data size: 18787251785 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator diff --git a/ql/src/test/results/clientpositive/perf/spark/query42.q.out b/ql/src/test/results/clientpositive/perf/spark/query42.q.out index 73c0df1827..e1f0cdd763 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query42.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query42.q.out @@ -165,27 +165,23 @@ STAGE PLANS: Reducer 5 Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: decimal(17,2)) - outputColumnNames: _col0, _col1, _col2 + expressions: 1998 (type: int), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: decimal(17,2)) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 100 Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 1998 (type: int), _col0 (type: int), _col1 (type: string), _col2 (type: decimal(17,2)) - outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator - limit: -1 + limit: 100 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/perf/spark/query47.q.out b/ql/src/test/results/clientpositive/perf/spark/query47.q.out index def85ad627..a61afcdd7f 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query47.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query47.q.out @@ -624,14 +624,14 @@ STAGE PLANS: Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: decimal(17,2)), rank_window_0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col7 - 1) (type: int) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col5 - 1) (type: int) sort order: +++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col7 - 1) (type: int) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col5 - 1) (type: int) Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE - value expressions: _col6 (type: decimal(17,2)) + value expressions: _col4 (type: decimal(17,2)) Reducer 3 Local Work: Map Reduce Local Work @@ -716,14 +716,14 @@ STAGE PLANS: Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: decimal(17,2)), rank_window_0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col7 + 1) (type: int) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col5 + 1) (type: int) sort order: +++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col7 + 1) (type: int) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col5 + 1) (type: int) Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE - value expressions: _col6 (type: decimal(17,2)) + value expressions: _col4 (type: decimal(17,2)) Reducer 6 Reduce Operator Tree: Join Operator @@ -731,13 +731,13 @@ STAGE PLANS: Inner Join 0 to 1 Inner Join 1 to 2 keys: - 0 _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col7 + 1) (type: int) + 0 _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col5 + 1) (type: int) 1 _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col8 (type: int) - 2 _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col7 - 1) (type: int) - outputColumnNames: _col6, _col8, _col12, _col13, _col14, _col15, _col23 + 2 _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), (_col5 - 1) (type: int) + outputColumnNames: _col4, _col6, _col10, _col11, _col12, _col13, _col19 Statistics: Num rows: 843315280 Data size: 74397518857 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col8 (type: string), _col12 (type: int), _col13 (type: int), _col15 (type: decimal(21,6)), _col14 (type: decimal(17,2)), _col6 (type: decimal(17,2)), _col23 (type: decimal(17,2)), (_col14 - _col15) (type: decimal(22,6)) + expressions: _col6 (type: string), _col10 (type: int), _col11 (type: int), _col13 (type: decimal(21,6)), _col12 (type: decimal(17,2)), _col4 (type: decimal(17,2)), _col19 (type: decimal(17,2)), (_col12 - _col13) (type: decimal(22,6)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 843315280 Data size: 74397518857 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator diff --git a/ql/src/test/results/clientpositive/perf/spark/query57.q.out b/ql/src/test/results/clientpositive/perf/spark/query57.q.out index d6cb244ee8..b6d58c1022 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query57.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query57.q.out @@ -497,9 +497,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 15971437 Data size: 2162853414 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string), _col1 (type: string), _col7 (type: int) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col7 (type: int) sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string), _col1 (type: string), _col7 (type: int) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col7 (type: int) Statistics: Num rows: 15971437 Data size: 2162853414 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: int), _col4 (type: int), _col5 (type: decimal(17,2)), _col6 (type: decimal(21,6)) Reducer 2 @@ -630,14 +630,14 @@ STAGE PLANS: Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: decimal(17,2)), rank_window_0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string), _col1 (type: string), (_col6 - 1) (type: int) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), (_col4 - 1) (type: int) sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string), _col1 (type: string), (_col6 - 1) (type: int) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), (_col4 - 1) (type: int) Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE - value expressions: _col5 (type: decimal(17,2)) + value expressions: _col3 (type: decimal(17,2)) Reducer 3 Reduce Operator Tree: Join Operator @@ -710,14 +710,14 @@ STAGE PLANS: Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: decimal(17,2)), rank_window_0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string), _col1 (type: string), (_col6 + 1) (type: int) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), (_col4 + 1) (type: int) sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string), _col1 (type: string), (_col6 + 1) (type: int) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), (_col4 + 1) (type: int) Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE - value expressions: _col5 (type: decimal(17,2)) + value expressions: _col3 (type: decimal(17,2)) Reducer 6 Reduce Operator Tree: Join Operator @@ -725,13 +725,13 @@ STAGE PLANS: Inner Join 0 to 1 Inner Join 1 to 2 keys: - 0 _col0 (type: string), _col2 (type: string), _col1 (type: string), (_col6 + 1) (type: int) - 1 _col0 (type: string), _col2 (type: string), _col1 (type: string), _col7 (type: int) - 2 _col0 (type: string), _col2 (type: string), _col1 (type: string), (_col6 - 1) (type: int) - outputColumnNames: _col5, _col7, _col8, _col10, _col11, _col12, _col13, _col20 + 0 _col0 (type: string), _col1 (type: string), _col2 (type: string), (_col4 + 1) (type: int) + 1 _col0 (type: string), _col1 (type: string), _col2 (type: string), _col7 (type: int) + 2 _col0 (type: string), _col1 (type: string), _col2 (type: string), (_col4 - 1) (type: int) + outputColumnNames: _col3, _col5, _col6, _col8, _col9, _col10, _col11, _col16 Statistics: Num rows: 421645952 Data size: 57099332264 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: string), _col8 (type: string), _col10 (type: int), _col11 (type: int), _col13 (type: decimal(21,6)), _col12 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col20 (type: decimal(17,2)), (_col12 - _col13) (type: decimal(22,6)) + expressions: _col5 (type: string), _col6 (type: string), _col8 (type: int), _col9 (type: int), _col11 (type: decimal(21,6)), _col10 (type: decimal(17,2)), _col3 (type: decimal(17,2)), _col16 (type: decimal(17,2)), (_col10 - _col11) (type: decimal(22,6)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 421645952 Data size: 57099332264 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator diff --git a/ql/src/test/results/clientpositive/perf/spark/query74.q.out b/ql/src/test/results/clientpositive/perf/spark/query74.q.out index 24cb881036..3ae334e868 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query74.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query74.q.out @@ -411,12 +411,16 @@ STAGE PLANS: Filter Operator predicate: (_col4 > 0) (type: boolean) Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col4 (type: decimal(7,2)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE - value expressions: _col4 (type: decimal(7,2)) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(7,2)) Reducer 16 Reduce Operator Tree: Join Operator @@ -465,14 +469,14 @@ STAGE PLANS: Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col4 (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col4 (type: decimal(7,2)) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: decimal(7,2)) Reducer 2 Reduce Operator Tree: Join Operator @@ -537,14 +541,14 @@ STAGE PLANS: Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: decimal(7,2)) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col1 Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE - value expressions: _col4 (type: decimal(7,2)) + value expressions: _col1 (type: decimal(7,2)) Reducer 3 Reduce Operator Tree: Join Operator @@ -582,12 +586,16 @@ STAGE PLANS: Filter Operator predicate: (_col4 > 0) (type: boolean) Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col4 (type: decimal(7,2)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE - value expressions: _col4 (type: decimal(7,2)) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(7,2)) Reducer 5 Reduce Operator Tree: Join Operator @@ -600,13 +608,13 @@ STAGE PLANS: 1 _col0 (type: string) 2 _col0 (type: string) 3 _col0 (type: string) - outputColumnNames: _col4, _col9, _col10, _col11, _col12, _col14, _col19 + outputColumnNames: _col1, _col3, _col4, _col5, _col6, _col7, _col9 Statistics: Num rows: 1149975359 Data size: 101451160012 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col19 / _col4) > (_col14 / _col9)) (type: boolean) + predicate: ((_col9 / _col1) > (_col7 / _col3)) (type: boolean) Statistics: Num rows: 383325119 Data size: 33817053278 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col10 (type: string), _col11 (type: string), _col12 (type: string) + expressions: _col4 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 383325119 Data size: 33817053278 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator diff --git a/ql/src/test/results/clientpositive/perf/spark/query78.q.out b/ql/src/test/results/clientpositive/perf/spark/query78.q.out index 651348cf84..1467c5f155 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query78.q.out @@ -474,23 +474,19 @@ STAGE PLANS: Reducer 6 Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey8 (type: double), KEY.reducesinkkey2 (type: bigint), KEY.reducesinkkey3 (type: decimal(17,2)), KEY.reducesinkkey4 (type: decimal(17,2)), KEY.reducesinkkey5 (type: bigint), KEY.reducesinkkey6 (type: decimal(18,2)), KEY.reducesinkkey7 (type: decimal(18,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + expressions: 2000 (type: int), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey8 (type: double), KEY.reducesinkkey2 (type: bigint), KEY.reducesinkkey3 (type: decimal(17,2)), KEY.reducesinkkey4 (type: decimal(17,2)), KEY.reducesinkkey5 (type: bigint), KEY.reducesinkkey6 (type: decimal(18,2)), KEY.reducesinkkey7 (type: decimal(18,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 23425424 Data size: 2066597727 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 100 Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 2000 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: double), _col3 (type: bigint), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col6 (type: bigint), _col7 (type: decimal(18,2)), _col8 (type: decimal(18,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + File Output Operator + compressed: false Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 8 Reduce Operator Tree: Join Operator @@ -517,7 +513,7 @@ STAGE PLANS: Stage: Stage-0 Fetch Operator - limit: -1 + limit: 100 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/perf/spark/query85.q.out b/ql/src/test/results/clientpositive/perf/spark/query85.q.out index 2035de016c..b4a4990a52 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query85.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query85.q.out @@ -461,10 +461,10 @@ STAGE PLANS: Statistics: Num rows: 1023990 Data size: 394452395 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: double), _col2 (type: decimal(11,6)), _col3 (type: decimal(11,6)), substr(_col0, 1, 20) (type: string) - outputColumnNames: _col5, _col6, _col7, _col8 + outputColumnNames: _col4, _col5, _col6, _col7 Statistics: Num rows: 1023990 Data size: 394452395 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col8 (type: string), _col5 (type: double), _col6 (type: decimal(11,6)), _col7 (type: decimal(11,6)) + key expressions: _col7 (type: string), _col4 (type: double), _col5 (type: decimal(11,6)), _col6 (type: decimal(11,6)) sort order: ++++ Statistics: Num rows: 1023990 Data size: 394452395 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 diff --git a/ql/src/test/results/clientpositive/perf/spark/query91.q.out b/ql/src/test/results/clientpositive/perf/spark/query91.q.out index fce1a15d30..b4f0650c84 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query91.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query91.q.out @@ -302,10 +302,10 @@ STAGE PLANS: Statistics: Num rows: 58564004 Data size: 50366227250 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: decimal(17,2)) - outputColumnNames: _col0, _col1, _col2, _col6 + outputColumnNames: _col0, _col1, _col2, _col4 Statistics: Num rows: 58564004 Data size: 50366227250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col6 (type: decimal(17,2)) + key expressions: _col4 (type: decimal(17,2)) sort order: - Statistics: Num rows: 58564004 Data size: 50366227250 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) diff --git a/ql/src/test/results/clientpositive/perf/tez/query11.q.out b/ql/src/test/results/clientpositive/perf/tez/query11.q.out index 1dd63be54c..e3ac7172f3 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query11.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query11.q.out @@ -179,14 +179,14 @@ Stage-0 Select Operator [SEL_92] (rows=383325119 width=88) Output:["_col0"] Filter Operator [FIL_91] (rows=383325119 width=88) - predicate:((_col7 / _col30) > (_col22 / _col14)) + predicate:((_col1 / _col8) > (_col6 / _col3)) Merge Join Operator [MERGEJOIN_175] (rows=1149975359 width=88) - Conds:RS_86._col0=RS_87._col0(Inner),RS_87._col0=RS_88._col0(Inner),RS_87._col0=RS_89._col0(Inner),Output:["_col7","_col14","_col18","_col22","_col30"] + Conds:RS_86._col0=RS_87._col0(Inner),RS_87._col0=RS_88._col0(Inner),RS_87._col0=RS_89._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col8"] <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_88] PartitionCols:_col0 Select Operator [SEL_63] (rows=348477374 width=88) - Output:["_col0","_col3","_col7"] + Output:["_col0","_col1","_col2"] Group By Operator [GBY_62] (rows=348477374 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 <-Reducer 12 [SIMPLE_EDGE] @@ -234,7 +234,7 @@ Stage-0 SHUFFLE [RS_87] PartitionCols:_col0 Select Operator [SEL_42] (rows=116159124 width=88) - Output:["_col0","_col6"] + Output:["_col0","_col1"] Filter Operator [FIL_41] (rows=116159124 width=88) predicate:(_col7 > 0) Select Operator [SEL_166] (rows=348477374 width=88) @@ -275,7 +275,7 @@ Stage-0 SHUFFLE [RS_86] PartitionCols:_col0 Select Operator [SEL_20] (rows=87121617 width=135) - Output:["_col0","_col7"] + Output:["_col0","_col1"] Group By Operator [GBY_19] (rows=87121617 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 <-Reducer 3 [SIMPLE_EDGE] @@ -312,40 +312,42 @@ Stage-0 <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_89] PartitionCols:_col0 - Filter Operator [FIL_84] (rows=29040539 width=135) - predicate:(_col7 > 0) - Select Operator [SEL_165] (rows=87121617 width=135) - Output:["_col0","_col7"] - Group By Operator [GBY_83] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Group By Operator [GBY_81] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Select Operator [SEL_79] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_174] (rows=174243235 width=135) - Conds:RS_76._col1=RS_77._col0(Inner),Output:["_col2","_col3","_col7","_col8","_col9","_col10","_col11","_col12","_col13"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_51] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_173] (rows=158402938 width=135) - Conds:RS_73._col0=RS_74._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_163] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - Please refer to the previous TableScan [TS_46] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] + Select Operator [SEL_85] (rows=29040539 width=135) + Output:["_col0","_col1"] + Filter Operator [FIL_84] (rows=29040539 width=135) + predicate:(_col7 > 0) + Select Operator [SEL_165] (rows=87121617 width=135) + Output:["_col0","_col7"] + Group By Operator [GBY_83] (rows=87121617 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_82] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Group By Operator [GBY_81] (rows=174243235 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Select Operator [SEL_79] (rows=174243235 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + Merge Join Operator [MERGEJOIN_174] (rows=174243235 width=135) + Conds:RS_76._col1=RS_77._col0(Inner),Output:["_col2","_col3","_col7","_col8","_col9","_col10","_col11","_col12","_col13"] + <-Map 18 [SIMPLE_EDGE] + SHUFFLE [RS_77] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_51] + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_76] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_173] (rows=158402938 width=135) + Conds:RS_73._col0=RS_74._col0(Inner),Output:["_col1","_col2","_col3"] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_74] + PartitionCols:_col0 + Select Operator [SEL_69] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_163] (rows=36524 width=1119) + predicate:((d_year = 2001) and d_date_sk is not null) + Please refer to the previous TableScan [TS_46] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_73] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_2] diff --git a/ql/src/test/results/clientpositive/perf/tez/query39.q.out b/ql/src/test/results/clientpositive/perf/tez/query39.q.out index 38d5c263ba..793e375fb2 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query39.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query39.q.out @@ -74,122 +74,120 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_60] - Select Operator [SEL_59] (rows=13756683 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_104] (rows=13756683 width=15) - Conds:RS_56._col1, _col2=RS_57._col1, _col2(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col1, _col2 - Select Operator [SEL_55] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_54] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_53] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_52] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_50] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Merge Join Operator [MERGEJOIN_103] (rows=50024305 width=15) - Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_93] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_9] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_102] (rows=45476640 width=15) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_92] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_101] (rows=41342400 width=15) - Conds:RS_40._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_90] (rows=37584000 width=15) - predicate:(inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_33] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_95] (rows=18262 width=1119) - predicate:((d_moy = 5) and (d_year = 1999) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col1, _col2 - Select Operator [SEL_27] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_25] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_22] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Merge Join Operator [MERGEJOIN_100] (rows=50024305 width=15) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_11] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_99] (rows=45476640 width=15) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_8] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_98] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=18262 width=1119) - predicate:((d_moy = 4) and (d_year = 1999) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Merge Join Operator [MERGEJOIN_104] (rows=13756683 width=15) + Conds:RS_56._col0, _col1=RS_57._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + <-Reducer 11 [SIMPLE_EDGE] + SHUFFLE [RS_57] + PartitionCols:_col0, _col1 + Select Operator [SEL_55] (rows=12506076 width=15) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_54] (rows=12506076 width=15) + predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END + Select Operator [SEL_53] (rows=25012152 width=15) + Output:["_col1","_col2","_col3","_col4"] + Group By Operator [GBY_52] (rows=25012152 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Reducer 10 [SIMPLE_EDGE] + SHUFFLE [RS_51] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_50] (rows=50024305 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 + Merge Join Operator [MERGEJOIN_103] (rows=50024305 width=15) + Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col3","_col7","_col8","_col9"] + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_47] + PartitionCols:_col0 + Select Operator [SEL_11] (rows=27 width=1029) + Output:["_col0","_col1"] + Filter Operator [FIL_93] (rows=27 width=1029) + predicate:w_warehouse_sk is not null + TableScan [TS_9] (rows=27 width=1029) + default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] + <-Reducer 9 [SIMPLE_EDGE] + SHUFFLE [RS_46] + PartitionCols:_col2 + Merge Join Operator [MERGEJOIN_102] (rows=45476640 width=15) + Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col3","_col7"] + <-Map 13 [SIMPLE_EDGE] + SHUFFLE [RS_44] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=462000 width=1436) + Output:["_col0"] + Filter Operator [FIL_92] (rows=462000 width=1436) + predicate:i_item_sk is not null + TableScan [TS_6] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_43] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_101] (rows=41342400 width=15) + Conds:RS_40._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col3"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_40] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=37584000 width=15) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_90] (rows=37584000 width=15) + predicate:(inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) + TableScan [TS_0] (rows=37584000 width=15) + default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_41] + PartitionCols:_col0 + Select Operator [SEL_33] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_95] (rows=18262 width=1119) + predicate:((d_moy = 5) and (d_year = 1999) and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_56] + PartitionCols:_col0, _col1 + Select Operator [SEL_27] (rows=12506076 width=15) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_26] (rows=12506076 width=15) + predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END + Select Operator [SEL_25] (rows=25012152 width=15) + Output:["_col1","_col2","_col3","_col4"] + Group By Operator [GBY_24] (rows=25012152 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_23] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_22] (rows=50024305 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 + Merge Join Operator [MERGEJOIN_100] (rows=50024305 width=15) + Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col7","_col8","_col9"] + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_19] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_11] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_18] + PartitionCols:_col2 + Merge Join Operator [MERGEJOIN_99] (rows=45476640 width=15) + Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col7"] + <-Map 13 [SIMPLE_EDGE] + SHUFFLE [RS_16] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_8] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_15] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_98] (rows=41342400 width=15) + Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_2] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_13] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_91] (rows=18262 width=1119) + predicate:((d_moy = 4) and (d_year = 1999) and d_date_sk is not null) + Please refer to the previous TableScan [TS_3] PREHOOK: query: with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy diff --git a/ql/src/test/results/clientpositive/perf/tez/query4.q.out b/ql/src/test/results/clientpositive/perf/tez/query4.q.out index c56fae06bf..241c06390d 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query4.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query4.q.out @@ -253,14 +253,14 @@ Stage-0 Select Operator [SEL_138] (rows=212958399 width=88) Output:["_col0"] Filter Operator [FIL_136] (rows=212958399 width=88) - predicate:(((_col7 / _col31) > (_col23 / _col15)) and ((_col7 / _col31) > (_col39 / _col47))) + predicate:(((_col1 / _col8) > (_col10 / _col12)) and ((_col1 / _col8) > (_col6 / _col3))) Merge Join Operator [MERGEJOIN_296] (rows=1916625598 width=88) - Conds:RS_129._col0=RS_130._col0(Inner),RS_130._col0=RS_131._col0(Inner),RS_130._col0=RS_132._col0(Inner),RS_130._col0=RS_133._col0(Inner),RS_130._col0=RS_134._col0(Inner),Output:["_col7","_col15","_col19","_col23","_col31","_col39","_col47"] + Conds:RS_129._col0=RS_130._col0(Inner),RS_130._col0=RS_131._col0(Inner),RS_130._col0=RS_132._col0(Inner),RS_130._col0=RS_133._col0(Inner),RS_130._col0=RS_134._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col8","_col10","_col12"] <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_131] PartitionCols:_col0 Select Operator [SEL_63] (rows=348477374 width=88) - Output:["_col0","_col3","_col7"] + Output:["_col0","_col1","_col2"] Group By Operator [GBY_62] (rows=348477374 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 <-Reducer 12 [SIMPLE_EDGE] @@ -307,47 +307,49 @@ Stage-0 <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_130] PartitionCols:_col0 - Filter Operator [FIL_41] (rows=116159124 width=88) - predicate:(_col7 > 0) - Select Operator [SEL_283] (rows=348477374 width=88) - Output:["_col0","_col7"] - Group By Operator [GBY_40] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Group By Operator [GBY_38] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Select Operator [SEL_36] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_287] (rows=696954748 width=88) - Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_94] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_286] (rows=633595212 width=88) - Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_267] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - Please refer to the previous TableScan [TS_89] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_45] + Select Operator [SEL_42] (rows=116159124 width=88) + Output:["_col0","_col1"] + Filter Operator [FIL_41] (rows=116159124 width=88) + predicate:(_col7 > 0) + Select Operator [SEL_283] (rows=348477374 width=88) + Output:["_col0","_col7"] + Group By Operator [GBY_40] (rows=348477374 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_39] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Group By Operator [GBY_38] (rows=696954748 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Select Operator [SEL_36] (rows=696954748 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + Merge Join Operator [MERGEJOIN_287] (rows=696954748 width=88) + Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] + <-Map 25 [SIMPLE_EDGE] + SHUFFLE [RS_34] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_94] + <-Reducer 14 [SIMPLE_EDGE] + SHUFFLE [RS_33] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_286] (rows=633595212 width=88) + Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] + <-Map 24 [SIMPLE_EDGE] + SHUFFLE [RS_31] + PartitionCols:_col0 + Select Operator [SEL_26] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_267] (rows=36524 width=1119) + predicate:((d_year = 2001) and d_date_sk is not null) + Please refer to the previous TableScan [TS_89] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_30] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_45] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_133] PartitionCols:_col0 Select Operator [SEL_106] (rows=87121617 width=135) - Output:["_col0","_col7"] + Output:["_col0","_col1"] Group By Operator [GBY_105] (rows=87121617 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 <-Reducer 19 [SIMPLE_EDGE] @@ -384,47 +386,49 @@ Stage-0 <-Reducer 23 [SIMPLE_EDGE] SHUFFLE [RS_134] PartitionCols:_col0 - Filter Operator [FIL_127] (rows=29040539 width=135) - predicate:(_col7 > 0) - Select Operator [SEL_282] (rows=87121617 width=135) - Output:["_col0","_col7"] - Group By Operator [GBY_126] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_125] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Group By Operator [GBY_124] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Select Operator [SEL_122] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_295] (rows=174243235 width=135) - Conds:RS_119._col1=RS_120._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_120] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_94] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_294] (rows=158402938 width=135) - Conds:RS_116._col0=RS_117._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_117] - PartitionCols:_col0 - Select Operator [SEL_112] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_279] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - Please refer to the previous TableScan [TS_89] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_88] + Select Operator [SEL_128] (rows=29040539 width=135) + Output:["_col0","_col1"] + Filter Operator [FIL_127] (rows=29040539 width=135) + predicate:(_col7 > 0) + Select Operator [SEL_282] (rows=87121617 width=135) + Output:["_col0","_col7"] + Group By Operator [GBY_126] (rows=87121617 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 + <-Reducer 22 [SIMPLE_EDGE] + SHUFFLE [RS_125] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Group By Operator [GBY_124] (rows=174243235 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Select Operator [SEL_122] (rows=174243235 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + Merge Join Operator [MERGEJOIN_295] (rows=174243235 width=135) + Conds:RS_119._col1=RS_120._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] + <-Map 25 [SIMPLE_EDGE] + SHUFFLE [RS_120] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_94] + <-Reducer 21 [SIMPLE_EDGE] + SHUFFLE [RS_119] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_294] (rows=158402938 width=135) + Conds:RS_116._col0=RS_117._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] + <-Map 24 [SIMPLE_EDGE] + SHUFFLE [RS_117] + PartitionCols:_col0 + Select Operator [SEL_112] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_279] (rows=36524 width=1119) + predicate:((d_year = 2001) and d_date_sk is not null) + Please refer to the previous TableScan [TS_89] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_116] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_88] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_129] PartitionCols:_col0 Select Operator [SEL_20] (rows=174233858 width=135) - Output:["_col0","_col7"] + Output:["_col0","_col1"] Group By Operator [GBY_19] (rows=174233858 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 <-Reducer 3 [SIMPLE_EDGE] @@ -461,40 +465,42 @@ Stage-0 <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_132] PartitionCols:_col0 - Filter Operator [FIL_84] (rows=58077952 width=135) - predicate:(_col7 > 0) - Select Operator [SEL_281] (rows=174233858 width=135) - Output:["_col0","_col7"] - Group By Operator [GBY_83] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Group By Operator [GBY_81] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 - Select Operator [SEL_79] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_291] (rows=348467716 width=135) - Conds:RS_76._col1=RS_77._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_94] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_290] (rows=316788826 width=135) - Conds:RS_73._col0=RS_74._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_273] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - Please refer to the previous TableScan [TS_89] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] + Select Operator [SEL_85] (rows=58077952 width=135) + Output:["_col0","_col1"] + Filter Operator [FIL_84] (rows=58077952 width=135) + predicate:(_col7 > 0) + Select Operator [SEL_281] (rows=174233858 width=135) + Output:["_col0","_col7"] + Group By Operator [GBY_83] (rows=174233858 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6 + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_82] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Group By Operator [GBY_81] (rows=348467716 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6 + Select Operator [SEL_79] (rows=348467716 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + Merge Join Operator [MERGEJOIN_291] (rows=348467716 width=135) + Conds:RS_76._col1=RS_77._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] + <-Map 25 [SIMPLE_EDGE] + SHUFFLE [RS_77] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_94] + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_76] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_290] (rows=316788826 width=135) + Conds:RS_73._col0=RS_74._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] + <-Map 24 [SIMPLE_EDGE] + SHUFFLE [RS_74] + PartitionCols:_col0 + Select Operator [SEL_69] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_273] (rows=36524 width=1119) + predicate:((d_year = 2001) and d_date_sk is not null) + Please refer to the previous TableScan [TS_89] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_73] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_2] diff --git a/ql/src/test/results/clientpositive/perf/tez/query42.q.out b/ql/src/test/results/clientpositive/perf/tez/query42.q.out index f31a62dfc9..0e07c4d93a 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query42.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query42.q.out @@ -50,59 +50,57 @@ Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-0 Fetch Operator - limit:-1 + limit:100 Stage-1 Reducer 5 File Output Operator [FS_24] - Select Operator [SEL_23] (rows=100 width=88) - Output:["_col0","_col1","_col2","_col3"] - Limit [LIM_22] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_21] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col3"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category_id","i_category","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Limit [LIM_23] (rows=100 width=88) + Number of rows:100 + Select Operator [SEL_21] (rows=348477374 width=88) + Output:["_col0","_col1","_col2","_col3"] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_20] + Select Operator [SEL_19] (rows=348477374 width=88) + Output:["_col0","_col1","_col3"] + Group By Operator [GBY_18] (rows=348477374 width=88) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_17] + PartitionCols:_col0, _col1 + Group By Operator [GBY_16] (rows=696954748 width=88) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 + Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] + <-Map 7 [SIMPLE_EDGE] + SHUFFLE [RS_13] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=231000 width=1436) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_32] (rows=231000 width=1436) + predicate:((i_manager_id = 1) and i_item_sk is not null) + TableScan [TS_6] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category_id","i_category","i_manager_id"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) + Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_9] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=575995635 width=88) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_30] (rows=575995635 width=88) + predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) + TableScan [TS_0] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] + <-Map 6 [SIMPLE_EDGE] + SHUFFLE [RS_10] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_31] (rows=18262 width=1119) + predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] diff --git a/ql/src/test/results/clientpositive/perf/tez/query47.q.out b/ql/src/test/results/clientpositive/perf/tez/query47.q.out index 04354b9d5d..a04269113e 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query47.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query47.q.out @@ -127,7 +127,7 @@ Stage-0 Select Operator [SEL_107] (rows=843315280 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Merge Join Operator [MERGEJOIN_189] (rows=843315280 width=88) - Conds:RS_103._col0, _col1, _col2, _col3, (_col7 + 1)=RS_104._col0, _col1, _col2, _col3, _col8(Inner),RS_104._col0, _col1, _col2, _col3, _col8=RS_105._col0, _col1, _col2, _col3, (_col7 - 1)(Inner),Output:["_col6","_col8","_col12","_col13","_col14","_col15","_col23"] + Conds:RS_103._col0, _col1, _col2, _col3, (_col5 + 1)=RS_104._col0, _col1, _col2, _col3, _col8(Inner),RS_104._col0, _col1, _col2, _col3, _col8=RS_105._col0, _col1, _col2, _col3, (_col5 - 1)(Inner),Output:["_col4","_col6","_col10","_col11","_col12","_col13","_col19"] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_104] PartitionCols:_col0, _col1, _col2, _col3, _col8 @@ -214,9 +214,9 @@ Stage-0 default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_105] - PartitionCols:_col0, _col1, _col2, _col3, (_col7 - 1) + PartitionCols:_col0, _col1, _col2, _col3, (_col5 - 1) Select Operator [SEL_99] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col6","_col7"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_175] (rows=383325119 width=88) predicate:rank_window_0 is not null PTF Operator [PTF_98] (rows=383325119 width=88) @@ -229,9 +229,9 @@ Stage-0 Please refer to the previous Select Operator [SEL_95] <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_103] - PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1) + PartitionCols:_col0, _col1, _col2, _col3, (_col5 + 1) Select Operator [SEL_29] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col6","_col7"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_164] (rows=383325119 width=88) predicate:rank_window_0 is not null PTF Operator [PTF_28] (rows=383325119 width=88) diff --git a/ql/src/test/results/clientpositive/perf/tez/query57.q.out b/ql/src/test/results/clientpositive/perf/tez/query57.q.out index 7a275d22c4..985d588572 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query57.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query57.q.out @@ -121,10 +121,10 @@ Stage-0 Select Operator [SEL_107] (rows=421645952 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] Merge Join Operator [MERGEJOIN_189] (rows=421645952 width=135) - Conds:RS_103._col0, _col2, _col1, (_col6 + 1)=RS_104._col0, _col2, _col1, _col7(Inner),RS_104._col0, _col2, _col1, _col7=RS_105._col0, _col2, _col1, (_col6 - 1)(Inner),Output:["_col5","_col7","_col8","_col10","_col11","_col12","_col13","_col20"] + Conds:RS_103._col0, _col1, _col2, (_col4 + 1)=RS_104._col0, _col1, _col2, _col7(Inner),RS_104._col0, _col1, _col2, _col7=RS_105._col0, _col1, _col2, (_col4 - 1)(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col10","_col11","_col16"] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_104] - PartitionCols:_col0, _col2, _col1, _col7 + PartitionCols:_col0, _col1, _col2, _col7 Select Operator [SEL_67] (rows=15971437 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Filter Operator [FIL_169] (rows=15971437 width=135) @@ -208,9 +208,9 @@ Stage-0 default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_105] - PartitionCols:_col0, _col2, _col1, (_col6 - 1) + PartitionCols:_col0, _col1, _col2, (_col4 - 1) Select Operator [SEL_99] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4"] Filter Operator [FIL_175] (rows=191657247 width=135) predicate:rank_window_0 is not null PTF Operator [PTF_98] (rows=191657247 width=135) @@ -223,9 +223,9 @@ Stage-0 Please refer to the previous Select Operator [SEL_95] <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_103] - PartitionCols:_col0, _col2, _col1, (_col6 + 1) + PartitionCols:_col0, _col1, _col2, (_col4 + 1) Select Operator [SEL_29] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4"] Filter Operator [FIL_164] (rows=191657247 width=135) predicate:rank_window_0 is not null PTF Operator [PTF_28] (rows=191657247 width=135) diff --git a/ql/src/test/results/clientpositive/perf/tez/query64.q.out b/ql/src/test/results/clientpositive/perf/tez/query64.q.out index cd76cc648b..26ba737797 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query64.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query64.q.out @@ -282,14 +282,14 @@ Stage-0 Select Operator [SEL_260] (rows=273897192 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] Filter Operator [FIL_259] (rows=273897192 width=88) - predicate:(_col30 <= _col13) + predicate:(_col19 <= _col12) Merge Join Operator [MERGEJOIN_615] (rows=821691577 width=88) - Conds:RS_256._col2, _col1, _col3=RS_257._col2, _col1, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col30","_col31","_col32","_col33"] + Conds:RS_256._col2, _col1, _col3=RS_257._col1, _col0, _col2(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col19","_col20","_col21","_col22"] <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_257] - PartitionCols:_col2, _col1, _col3 + PartitionCols:_col1, _col0, _col2 Select Operator [SEL_254] (rows=746992327 width=88) - Output:["_col1","_col2","_col3","_col13","_col14","_col15","_col16"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] Group By Operator [GBY_253] (rows=746992327 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 <-Reducer 13 [SIMPLE_EDGE] @@ -545,7 +545,7 @@ Stage-0 SHUFFLE [RS_256] PartitionCols:_col2, _col1, _col3 Select Operator [SEL_126] (rows=746992327 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] Group By Operator [GBY_125] (rows=746992327 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 <-Reducer 8 [SIMPLE_EDGE] diff --git a/ql/src/test/results/clientpositive/perf/tez/query74.q.out b/ql/src/test/results/clientpositive/perf/tez/query74.q.out index a0f5082cbd..ee5971850d 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query74.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query74.q.out @@ -151,14 +151,14 @@ Stage-0 Select Operator [SEL_88] (rows=383325119 width=88) Output:["_col0","_col1","_col2"] Filter Operator [FIL_87] (rows=383325119 width=88) - predicate:((_col19 / _col4) > (_col14 / _col9)) + predicate:((_col9 / _col1) > (_col7 / _col3)) Merge Join Operator [MERGEJOIN_171] (rows=1149975359 width=88) - Conds:RS_82._col0=RS_83._col0(Inner),RS_83._col0=RS_84._col0(Inner),RS_83._col0=RS_85._col0(Inner),Output:["_col4","_col9","_col10","_col11","_col12","_col14","_col19"] + Conds:RS_82._col0=RS_83._col0(Inner),RS_83._col0=RS_84._col0(Inner),RS_83._col0=RS_85._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col9"] <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_85] PartitionCols:_col0 Select Operator [SEL_81] (rows=87121617 width=135) - Output:["_col0","_col4"] + Output:["_col0","_col1"] Group By Operator [GBY_80] (rows=87121617 width=135) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 <-Reducer 12 [SIMPLE_EDGE] @@ -203,45 +203,47 @@ Stage-0 <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 - Filter Operator [FIL_19] (rows=29040539 width=135) - predicate:(_col4 > 0) - Select Operator [SEL_161] (rows=87121617 width=135) - Output:["_col0","_col4"] - Group By Operator [GBY_18] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_16] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4 - Merge Join Operator [MERGEJOIN_164] (rows=174243235 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_70] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_163] (rows=158402938 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=18262 width=1119) - predicate:((d_year = 2001) and (d_year) IN (2001, 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_65] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_64] + Select Operator [SEL_20] (rows=29040539 width=135) + Output:["_col0","_col1"] + Filter Operator [FIL_19] (rows=29040539 width=135) + predicate:(_col4 > 0) + Select Operator [SEL_161] (rows=87121617 width=135) + Output:["_col0","_col4"] + Group By Operator [GBY_18] (rows=87121617 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_17] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_16] (rows=174243235 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4 + Merge Join Operator [MERGEJOIN_164] (rows=174243235 width=135) + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"] + <-Map 18 [SIMPLE_EDGE] + SHUFFLE [RS_13] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_70] + <-Reducer 14 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_163] (rows=158402938 width=135) + Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2","_col4"] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_10] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=18262 width=1119) + Output:["_col0","_col1"] + Filter Operator [FIL_150] (rows=18262 width=1119) + predicate:((d_year = 2001) and (d_year) IN (2001, 2002) and d_date_sk is not null) + Please refer to the previous TableScan [TS_65] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_9] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_64] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_84] PartitionCols:_col0 Select Operator [SEL_61] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col4"] + Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_60] (rows=348477374 width=88) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 <-Reducer 3 [SIMPLE_EDGE] @@ -276,38 +278,40 @@ Stage-0 <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_83] PartitionCols:_col0 - Filter Operator [FIL_40] (rows=116159124 width=88) - predicate:(_col4 > 0) - Select Operator [SEL_162] (rows=348477374 width=88) - Output:["_col0","_col4"] - Group By Operator [GBY_39] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_37] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4 - Merge Join Operator [MERGEJOIN_166] (rows=696954748 width=88) - Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_70] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_165] (rows=633595212 width=88) - Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=18262 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=18262 width=1119) - predicate:((d_year = 2001) and (d_year) IN (2001, 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_65] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_44] + Select Operator [SEL_41] (rows=116159124 width=88) + Output:["_col0","_col1"] + Filter Operator [FIL_40] (rows=116159124 width=88) + predicate:(_col4 > 0) + Select Operator [SEL_162] (rows=348477374 width=88) + Output:["_col0","_col4"] + Group By Operator [GBY_39] (rows=348477374 width=88) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_38] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_37] (rows=696954748 width=88) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4 + Merge Join Operator [MERGEJOIN_166] (rows=696954748 width=88) + Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"] + <-Map 18 [SIMPLE_EDGE] + SHUFFLE [RS_34] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_70] + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_33] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_165] (rows=633595212 width=88) + Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col4"] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_31] + PartitionCols:_col0 + Select Operator [SEL_26] (rows=18262 width=1119) + Output:["_col0","_col1"] + Filter Operator [FIL_153] (rows=18262 width=1119) + predicate:((d_year = 2001) and (d_year) IN (2001, 2002) and d_date_sk is not null) + Please refer to the previous TableScan [TS_65] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_30] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_44] diff --git a/ql/src/test/results/clientpositive/perf/tez/query78.q.out b/ql/src/test/results/clientpositive/perf/tez/query78.q.out index 91c244fda6..9c2d7b3491 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query78.q.out @@ -130,163 +130,161 @@ Reducer 9 <- Map 1 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) Stage-0 Fetch Operator - limit:-1 + limit:100 Stage-1 Reducer 6 File Output Operator [FS_78] - Select Operator [SEL_77] (rows=100 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Limit [LIM_76] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_75] (rows=23425424 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_74] - Select Operator [SEL_73] (rows=23425424 width=88) - Output:["_col0","_col1","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Filter Operator [FIL_72] (rows=23425424 width=88) - predicate:(COALESCE(_col11,0) > 0) - Merge Join Operator [MERGEJOIN_113] (rows=70276272 width=88) - Conds:RS_69._col1=RS_70._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col7","_col8","_col9","_col11","_col12","_col13"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_67] (rows=43558464 width=135) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_66] (rows=43558464 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Group By Operator [GBY_64] (rows=87116928 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col3, _col4 - Merge Join Operator [MERGEJOIN_111] (rows=87116928 width=135) - Conds:RS_60._col0=RS_61._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_97] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_59] (rows=79197206 width=135) - Output:["_col0","_col1","_col2","_col4","_col5","_col6"] - Filter Operator [FIL_58] (rows=79197206 width=135) - predicate:_col8 is null - Merge Join Operator [MERGEJOIN_110] (rows=158394413 width=135) - Conds:RS_55._col2, _col3=RS_56._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col2, _col3 - Select Operator [SEL_52] (rows=143994918 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_104] (rows=143994918 width=135) - predicate:((cs_item_sk = cs_item_sk) and cs_sold_date_sk is not null) - TableScan [TS_50] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_order_number","cs_quantity","cs_wholesale_cost","cs_sales_price"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0, _col1 - Select Operator [SEL_54] (rows=28798881 width=106) - Output:["_col0","_col1"] - TableScan [TS_53] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Filter Operator [FIL_45] (rows=63887519 width=88) - predicate:(COALESCE(_col7,0) > 0) - Merge Join Operator [MERGEJOIN_112] (rows=191662559 width=88) - Conds:RS_42._col1, _col0=RS_43._col1, _col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col7","_col8","_col9"] - <-Reducer 3 [ONE_TO_ONE_EDGE] - FORWARD [RS_42] - PartitionCols:_col1, _col0 - Select Operator [SEL_20] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_19] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col4, _col3 - Merge Join Operator [MERGEJOIN_107] (rows=348477374 width=88) - Conds:RS_13._col0=RS_14._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_14] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=316797606 width=88) - Output:["_col0","_col1","_col2","_col4","_col5","_col6"] - Filter Operator [FIL_11] (rows=316797606 width=88) - predicate:_col8 is null - Merge Join Operator [MERGEJOIN_106] (rows=633595212 width=88) - Conds:RS_8._col1, _col3=RS_9._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_8] - PartitionCols:_col1, _col3 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_98] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Select Operator [SEL_7] (rows=57591150 width=77) - Output:["_col0","_col1"] - TableScan [TS_6] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 8 [ONE_TO_ONE_EDGE] - FORWARD [RS_43] - PartitionCols:_col1, _col0 - Select Operator [SEL_41] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_40] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0, _col1 - Group By Operator [GBY_38] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col4, _col3 - Merge Join Operator [MERGEJOIN_109] (rows=87121617 width=135) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_33] (rows=79201469 width=135) - Output:["_col0","_col1","_col2","_col4","_col5","_col6"] - Filter Operator [FIL_32] (rows=79201469 width=135) - predicate:_col8 is null - Merge Join Operator [MERGEJOIN_108] (rows=158402938 width=135) - Conds:RS_29._col1, _col3=RS_30._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col1, _col3 - Select Operator [SEL_26] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_101] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_24] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_order_number","ws_quantity","ws_wholesale_cost","ws_sales_price"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1 - Select Operator [SEL_28] (rows=14398467 width=92) - Output:["_col0","_col1"] - TableScan [TS_27] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number"] + Limit [LIM_77] (rows=100 width=88) + Number of rows:100 + Select Operator [SEL_75] (rows=23425424 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_74] + Select Operator [SEL_73] (rows=23425424 width=88) + Output:["_col0","_col1","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] + Filter Operator [FIL_72] (rows=23425424 width=88) + predicate:(COALESCE(_col11,0) > 0) + Merge Join Operator [MERGEJOIN_113] (rows=70276272 width=88) + Conds:RS_69._col1=RS_70._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col7","_col8","_col9","_col11","_col12","_col13"] + <-Reducer 10 [SIMPLE_EDGE] + SHUFFLE [RS_70] + PartitionCols:_col0 + Select Operator [SEL_67] (rows=43558464 width=135) + Output:["_col0","_col1","_col2","_col3"] + Group By Operator [GBY_66] (rows=43558464 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1 + <-Reducer 9 [SIMPLE_EDGE] + SHUFFLE [RS_65] + PartitionCols:_col0, _col1 + Group By Operator [GBY_64] (rows=87116928 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col3, _col4 + Merge Join Operator [MERGEJOIN_111] (rows=87116928 width=135) + Conds:RS_60._col0=RS_61._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_60] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_97] (rows=36524 width=1119) + predicate:((d_year = 2000) and d_date_sk is not null) + TableScan [TS_0] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + <-Reducer 18 [SIMPLE_EDGE] + SHUFFLE [RS_61] + PartitionCols:_col0 + Select Operator [SEL_59] (rows=79197206 width=135) + Output:["_col0","_col1","_col2","_col4","_col5","_col6"] + Filter Operator [FIL_58] (rows=79197206 width=135) + predicate:_col8 is null + Merge Join Operator [MERGEJOIN_110] (rows=158394413 width=135) + Conds:RS_55._col2, _col3=RS_56._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_55] + PartitionCols:_col2, _col3 + Select Operator [SEL_52] (rows=143994918 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_104] (rows=143994918 width=135) + predicate:((cs_item_sk = cs_item_sk) and cs_sold_date_sk is not null) + TableScan [TS_50] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_order_number","cs_quantity","cs_wholesale_cost","cs_sales_price"] + <-Map 19 [SIMPLE_EDGE] + SHUFFLE [RS_56] + PartitionCols:_col0, _col1 + Select Operator [SEL_54] (rows=28798881 width=106) + Output:["_col0","_col1"] + TableScan [TS_53] (rows=28798881 width=106) + default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number"] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_69] + PartitionCols:_col1 + Filter Operator [FIL_45] (rows=63887519 width=88) + predicate:(COALESCE(_col7,0) > 0) + Merge Join Operator [MERGEJOIN_112] (rows=191662559 width=88) + Conds:RS_42._col1, _col0=RS_43._col1, _col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col7","_col8","_col9"] + <-Reducer 3 [ONE_TO_ONE_EDGE] + FORWARD [RS_42] + PartitionCols:_col1, _col0 + Select Operator [SEL_20] (rows=174238687 width=88) + Output:["_col0","_col1","_col2","_col3","_col4"] + Group By Operator [GBY_19] (rows=174238687 width=88) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1 + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_18] + PartitionCols:_col0, _col1 + Group By Operator [GBY_17] (rows=348477374 width=88) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col4, _col3 + Merge Join Operator [MERGEJOIN_107] (rows=348477374 width=88) + Conds:RS_13._col0=RS_14._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_13] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_2] + <-Reducer 12 [SIMPLE_EDGE] + SHUFFLE [RS_14] + PartitionCols:_col0 + Select Operator [SEL_12] (rows=316797606 width=88) + Output:["_col0","_col1","_col2","_col4","_col5","_col6"] + Filter Operator [FIL_11] (rows=316797606 width=88) + predicate:_col8 is null + Merge Join Operator [MERGEJOIN_106] (rows=633595212 width=88) + Conds:RS_8._col1, _col3=RS_9._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_8] + PartitionCols:_col1, _col3 + Select Operator [SEL_5] (rows=575995635 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_98] (rows=575995635 width=88) + predicate:ss_sold_date_sk is not null + TableScan [TS_3] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_sales_price"] + <-Map 13 [SIMPLE_EDGE] + SHUFFLE [RS_9] + PartitionCols:_col0, _col1 + Select Operator [SEL_7] (rows=57591150 width=77) + Output:["_col0","_col1"] + TableScan [TS_6] (rows=57591150 width=77) + default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] + <-Reducer 8 [ONE_TO_ONE_EDGE] + FORWARD [RS_43] + PartitionCols:_col1, _col0 + Select Operator [SEL_41] (rows=43560808 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"] + Group By Operator [GBY_40] (rows=43560808 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1 + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_39] + PartitionCols:_col0, _col1 + Group By Operator [GBY_38] (rows=87121617 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col4, _col3 + Merge Join Operator [MERGEJOIN_109] (rows=87121617 width=135) + Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_34] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_2] + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_35] + PartitionCols:_col0 + Select Operator [SEL_33] (rows=79201469 width=135) + Output:["_col0","_col1","_col2","_col4","_col5","_col6"] + Filter Operator [FIL_32] (rows=79201469 width=135) + predicate:_col8 is null + Merge Join Operator [MERGEJOIN_108] (rows=158402938 width=135) + Conds:RS_29._col1, _col3=RS_30._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_29] + PartitionCols:_col1, _col3 + Select Operator [SEL_26] (rows=144002668 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_101] (rows=144002668 width=135) + predicate:ws_sold_date_sk is not null + TableScan [TS_24] (rows=144002668 width=135) + default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_order_number","ws_quantity","ws_wholesale_cost","ws_sales_price"] + <-Map 16 [SIMPLE_EDGE] + SHUFFLE [RS_30] + PartitionCols:_col0, _col1 + Select Operator [SEL_28] (rows=14398467 width=92) + Output:["_col0","_col1"] + TableScan [TS_27] (rows=14398467 width=92) + default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number"] diff --git a/ql/src/test/results/clientpositive/perf/tez/query85.q.out b/ql/src/test/results/clientpositive/perf/tez/query85.q.out index 90cc27e0b1..abba10da7a 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query85.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query85.q.out @@ -190,7 +190,7 @@ Stage-0 <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_52] Select Operator [SEL_51] (rows=1023990 width=385) - Output:["_col5","_col6","_col7","_col8"] + Output:["_col4","_col5","_col6","_col7"] Group By Operator [GBY_50] (rows=1023990 width=385) Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)"],keys:KEY._col0 <-Reducer 8 [SIMPLE_EDGE] diff --git a/ql/src/test/results/clientpositive/perf/tez/query91.q.out b/ql/src/test/results/clientpositive/perf/tez/query91.q.out index f808a50bb6..dc2e684a50 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query91.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query91.q.out @@ -81,7 +81,7 @@ Stage-0 <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_45] Select Operator [SEL_44] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col6"] + Output:["_col0","_col1","_col2","_col4"] Group By Operator [GBY_43] (rows=58564004 width=860) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 <-Reducer 4 [SIMPLE_EDGE] diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out index 2eb5db1d20..68889d306f 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out @@ -79,35 +79,30 @@ STAGE PLANS: Map 2 Map Operator Tree: TableScan - alias: b + alias: c Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: key is not null (type: boolean) + predicate: value is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + expressions: value (type: string) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 _col1 (type: string) + 1 _col0 (type: string) Position of Big Table: 0 Local Work: Map Reduce Local Work - Bucket Mapjoin Context: - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition - base file name: tbl2 + base file name: tbl3 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -120,11 +115,11 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl2 + name default.tbl3 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl2 { i32 key, string value} + serialization.ddl struct tbl3 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 @@ -143,38 +138,38 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl2 + name default.tbl3 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl2 { i32 key, string value} + serialization.ddl struct tbl3 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tbl2 - name: default.tbl2 + name: default.tbl3 + name: default.tbl3 Truncated Path -> Alias: - /tbl2 [$hdt$_1:b] + /tbl3 [$hdt$_1:c] Map 3 Map Operator Tree: TableScan - alias: c + alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: value is not null (type: boolean) + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) - outputColumnNames: _col1 + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: int) + 1 _col0 (type: int) Position of Big Table: 0 Local Work: Map Reduce Local Work @@ -183,7 +178,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: tbl3 + base file name: tbl2 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -196,11 +191,11 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl3 + name default.tbl2 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl3 { i32 key, string value} + serialization.ddl struct tbl2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 @@ -219,20 +214,20 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl3 + name default.tbl2 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl3 { i32 key, string value} + serialization.ddl struct tbl2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tbl3 - name: default.tbl3 + name: default.tbl2 + name: default.tbl2 Truncated Path -> Alias: - /tbl3 [$hdt$_2:c] + /tbl2 [$hdt$_2:b] Stage: Stage-1 Spark @@ -256,27 +251,26 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 input vertices: 1 Map 2 Position of Big Table: 0 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col3, _col5 + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col4 input vertices: 1 Map 3 Position of Big Table: 0 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col5 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -303,11 +297,6 @@ STAGE PLANS: MultiFileSpray: false Local Work: Map Reduce Local Work - Bucket Mapjoin Context: - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -463,21 +452,21 @@ STAGE PLANS: Map 2 Map Operator Tree: TableScan - alias: b + alias: c Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: key is not null (type: boolean) + predicate: value is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + expressions: value (type: string) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 _col1 (type: string) + 1 _col0 (type: string) Position of Big Table: 0 Local Work: Map Reduce Local Work @@ -486,7 +475,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: tbl2 + base file name: tbl3 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -499,11 +488,11 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl2 + name default.tbl3 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl2 { i32 key, string value} + serialization.ddl struct tbl3 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 @@ -522,38 +511,38 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl2 + name default.tbl3 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl2 { i32 key, string value} + serialization.ddl struct tbl3 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tbl2 - name: default.tbl2 + name: default.tbl3 + name: default.tbl3 Truncated Path -> Alias: - /tbl2 [$hdt$_1:b] + /tbl3 [$hdt$_1:c] Map 3 Map Operator Tree: TableScan - alias: c + alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: value is not null (type: boolean) + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) - outputColumnNames: _col1 + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 _col0 (type: int) + 1 _col0 (type: int) Position of Big Table: 0 Local Work: Map Reduce Local Work @@ -562,7 +551,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: tbl3 + base file name: tbl2 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -575,11 +564,11 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl3 + name default.tbl2 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl3 { i32 key, string value} + serialization.ddl struct tbl2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 @@ -598,20 +587,20 @@ STAGE PLANS: columns.comments columns.types int:string #### A masked pattern was here #### - name default.tbl3 + name default.tbl2 numFiles 2 numRows 10 rawDataSize 70 - serialization.ddl struct tbl3 { i32 key, string value} + serialization.ddl struct tbl2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 80 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tbl3 - name: default.tbl3 + name: default.tbl2 + name: default.tbl2 Truncated Path -> Alias: - /tbl3 [$hdt$_2:c] + /tbl2 [$hdt$_2:b] Stage: Stage-1 Spark @@ -635,9 +624,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 input vertices: 1 Map 2 Position of Big Table: 0 @@ -646,15 +635,15 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col3, _col5 + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col4 input vertices: 1 Map 3 Position of Big Table: 0 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col5 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out b/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out index 39a5d5d21e..d3439e42c2 100644 --- a/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out +++ b/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out @@ -954,14 +954,14 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col3 / _col4)) END (type: double) - outputColumnNames: _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int), _col2 (type: int) + key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ - Map-reduce partition columns: _col1 (type: int), _col2 (type: int) + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double), _col4 (type: double) + value expressions: _col2 (type: double), _col3 (type: double) Reducer 2 Reduce Operator Tree: Join Operator @@ -1033,33 +1033,29 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col3 / _col4)) END (type: double) - outputColumnNames: _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int), _col2 (type: int) + key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ - Map-reduce partition columns: _col1 (type: int), _col2 (type: int) + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double), _col4 (type: double) + value expressions: _col2 (type: double), _col3 (type: double) Reducer 6 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: int), _col2 (type: int) - 1 _col1 (type: int), _col2 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col9 + 0 _col0 (type: int), _col1 (type: int) + 1 _col0 (type: int), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col3 (type: double), _col4 (type: double), _col6 (type: int), _col7 (type: int), _col8 (type: double), _col9 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: double), _col3 (type: double), _col6 (type: double), _col7 (type: double) + sort order: ++++++ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: double), _col3 (type: double), _col6 (type: double), _col7 (type: double) - sort order: ++++++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: int), _col5 (type: int) + value expressions: _col4 (type: int), _col5 (type: int) Reducer 7 Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out index b441f27df5..a60d8ddbe3 100644 --- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out @@ -1315,7 +1315,7 @@ STAGE PLANS: Stage: Stage-0 Fetch Operator - limit: -1 + limit: 100 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_4.q.out b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_4.q.out index e7a789ad65..685e874558 100644 --- a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_4.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_4.q.out @@ -1517,10 +1517,10 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 220 Data size: 2200 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col3 (type: string) + expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 220 Data size: 2200 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1535,10 +1535,10 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 220 Data size: 2200 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col3 (type: string) + expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 220 Data size: 2200 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1834,10 +1834,10 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 220 Data size: 2200 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col3 (type: string) + expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 220 Data size: 2200 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index 4afff7807a..ebd8d8692c 100644 --- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -1188,10 +1188,10 @@ Reducer 3 <- Reducer 2 (SORT) Stage-0 Fetch Operator - limit:-1 + limit:1 Stage-1 Reducer 3 - File Output Operator [FS_11] + File Output Operator [FS_10] Limit [LIM_9] (rows=1 width=97) Number of rows:1 Select Operator [SEL_8] (rows=10 width=97) @@ -1375,7 +1375,7 @@ Stage-0 <-Reducer 2 [SORT] SORT [RS_8] Select Operator [SEL_6] (rows=3 width=105) - Output:["_col0","_col1","_col2","_col5"] + Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_5] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 <-Map 1 [GROUP] diff --git a/ql/src/test/results/clientpositive/spark/subquery_notin.q.out b/ql/src/test/results/clientpositive/spark/subquery_notin.q.out index 9dbe342e50..b2a1972e15 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_notin.q.out @@ -6168,11 +6168,11 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - 2 _col3 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4, _col7 + 2 _col2 (type: int) + outputColumnNames: _col0, _col1, _col3, _col4, _col6 Statistics: Num rows: 4 Data size: 19 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col6 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int) @@ -6208,14 +6208,14 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: Join Operator @@ -6224,14 +6224,14 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int) + key expressions: _col2 (type: int) sort order: + - Map-reduce partition columns: _col3 (type: int) + Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) Reducer 9 Reduce Operator Tree: Group By Operator @@ -6396,11 +6396,11 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - 2 _col3 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4, _col7 + 2 _col2 (type: int) + outputColumnNames: _col0, _col1, _col3, _col4, _col6 Statistics: Num rows: 6 Data size: 22 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: (not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col6 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) @@ -6436,14 +6436,14 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: Join Operator @@ -6452,14 +6452,14 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int) + key expressions: _col2 (type: int) sort order: + - Map-reduce partition columns: _col3 (type: int) + Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) Reducer 9 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/subquery_views.q.out b/ql/src/test/results/clientpositive/spark/subquery_views.q.out index 815d6bb01a..14be0186dd 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_views.q.out @@ -232,11 +232,11 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - 2 _col4 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col4, _col5, _col9 + 2 _col3 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col4, _col5, _col8 Statistics: Num rows: 605 Data size: 6426 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col9 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 302 Data size: 3207 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -275,14 +275,14 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string), _col3 (type: boolean) + value expressions: _col1 (type: string), _col2 (type: boolean) Reducer 17 Reduce Operator Tree: Join Operator @@ -291,14 +291,14 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col2, _col3, _col4 + outputColumnNames: _col1, _col2, _col3 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: string), _col2 (type: string) + key expressions: _col3 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col4 (type: string), _col2 (type: string) + Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: boolean) + value expressions: _col2 (type: boolean) Reducer 19 Reduce Operator Tree: Group By Operator @@ -320,11 +320,11 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - 2 _col4 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5, _col9 + 2 _col3 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5, _col8 Statistics: Num rows: 605 Data size: 6426 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col9 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 302 Data size: 3207 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) @@ -376,14 +376,14 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string), _col3 (type: boolean) + value expressions: _col1 (type: string), _col2 (type: boolean) Reducer 8 Reduce Operator Tree: Join Operator @@ -392,14 +392,14 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col2, _col3, _col4 + outputColumnNames: _col1, _col2, _col3 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: string), _col2 (type: string) + key expressions: _col3 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col4 (type: string), _col2 (type: string) + Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.out index e8083572ec..74239031f6 100644 --- a/ql/src/test/results/clientpositive/subquery_notin_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.out @@ -1085,11 +1085,11 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: bigint) TableScan Reduce Output Operator - key expressions: _col3 (type: int) + key expressions: _col2 (type: int) sort order: + - Map-reduce partition columns: _col3 (type: int) + Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 2 Data size: 171 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) Reduce Operator Tree: Join Operator condition map: @@ -1098,11 +1098,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - 2 _col3 (type: int) - outputColumnNames: _col0, _col2, _col3, _col6 + 2 _col2 (type: int) + outputColumnNames: _col0, _col2, _col3, _col5 Statistics: Num rows: 4 Data size: 343 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col2 = 0)) THEN (false) WHEN (_col2 is null) THEN (false) WHEN (_col6 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col3 < _col2)) THEN (true) ELSE (false) END) (type: boolean) + predicate: (not CASE WHEN ((_col2 = 0)) THEN (false) WHEN (_col2 is null) THEN (false) WHEN (_col5 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col3 < _col2)) THEN (true) ELSE (false) END) (type: boolean) Statistics: Num rows: 2 Data size: 171 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) @@ -1178,7 +1178,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1196,7 +1196,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: boolean) + value expressions: _col1 (type: boolean) TableScan Reduce Output Operator key expressions: _col0 (type: int) @@ -1210,7 +1210,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col3 + outputColumnNames: _col1, _col2 Statistics: Num rows: 2 Data size: 171 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false