diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 3f51740..89243fc 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -455,6 +455,8 @@ spark.query.files=add_part_multiple.q \ order.q \ order2.q \ parallel.q \ + parallel_join0.q, \ + parallel_join1.q, \ ppd_multi_insert.q \ ppd_transform.q \ ptf_decimal.q \ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java index 446e3cc..2545a9d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java @@ -20,6 +20,8 @@ import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.io.BytesWritable; +import org.apache.spark.HashPartitioner; +import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.function.PairFlatMapFunction; import scala.Tuple2; @@ -28,14 +30,28 @@ public class SortByShuffler implements SparkShuffler { + private final boolean totalOrder; + + /** + * @param totalOrder whether this shuffler provides total order shuffle. + */ + public SortByShuffler(boolean totalOrder) { + this.totalOrder = totalOrder; + } + @Override public JavaPairRDD> shuffle( JavaPairRDD input, int numPartitions) { JavaPairRDD rdd; - if (numPartitions > 0) { - rdd = input.sortByKey(true, numPartitions); + if (totalOrder) { + if (numPartitions > 0) { + rdd = input.sortByKey(true, numPartitions); + } else { + rdd = input.sortByKey(true); + } } else { - rdd = input.sortByKey(true); + Partitioner partitioner = new HashPartitioner(numPartitions); + rdd = input.repartitionAndSortWithinPartitions(partitioner); } return rdd.mapPartitionsToPair(new ShuffleFunction()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java index e1e0cf3..15af0f9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java @@ -176,8 +176,10 @@ public SparkTran generate(BaseWork work, SparkEdgeProperty edge) throws Exceptio private SparkShuffler generate(SparkEdgeProperty edge) { Preconditions.checkArgument(!edge.isShuffleNone(), "AssertionError: SHUFFLE_NONE should only be used for UnionWork."); - if (edge.isShuffleSort()) { - return new SortByShuffler(); + if (edge.isMRShuffle()) { + return new SortByShuffler(false); + } else if (edge.isShuffleSort()) { + return new SortByShuffler(true); } return new GroupByShuffler(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index ed06a57..280edde 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -34,15 +34,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; -import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.UnionOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.*; import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -127,6 +119,11 @@ public ReduceWork createReduceWork(GenSparkProcContext context, Operator root edgeProp.setShuffleSort(); } + if (reduceWork.getReducer() instanceof JoinOperator) { + //reduce-side join + edgeProp.setMRShuffle(); + } + sparkWork.connect( context.preceedingWork, reduceWork, edgeProp); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java index 4f889db..ac94ea0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java @@ -35,12 +35,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.BaseWork; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.ReduceWork; -import org.apache.hadoop.hive.ql.plan.SparkEdgeProperty; -import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.UnionWork; +import org.apache.hadoop.hive.ql.plan.*; import com.google.common.base.Preconditions; @@ -282,6 +277,10 @@ public Object process(Node nd, Stack stack, if (!sortOrder.isEmpty() && GenSparkUtils.isSortNecessary(rs)) { edgeProp.setShuffleSort(); } + if (rWork.getReducer() instanceof JoinOperator) { + //reduce-side join + edgeProp.setMRShuffle(); + } sparkWork.connect(work, rWork, edgeProp); context.connectedReduceSinks.add(rs); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java index bdfef87..83e664b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java @@ -22,8 +22,9 @@ @Explain(displayName = "Edge Property") public class SparkEdgeProperty { public static long SHUFFLE_NONE = 0; // No shuffle is needed. For union only. - public static long SHUFFLE_GROUP = 1; // Shuffle, keys are coming together - public static long SHUFFLE_SORT = 2; // Shuffle, keys are sorted + public static long SHUFFLE_GROUP = 1; // HashPartition shuffle, keys are not sorted in any way. + public static long SHUFFLE_SORT = 2; // RangePartition shuffle, keys are total sorted. + public static long MR_SHUFFLE_SORT = 4; // HashPartition shuffle, keys are sorted by partition. private long edgeType; @@ -49,19 +50,27 @@ public void setShuffleNone() { public boolean isShuffleGroup() { return (edgeType & SHUFFLE_GROUP) != 0; } - + public void setShuffleGroup() { edgeType |= SHUFFLE_GROUP; } - - public boolean isShuffleSort() { - return (edgeType & SHUFFLE_SORT) != 0; + + public void setMRShuffle() { + edgeType |= MR_SHUFFLE_SORT; + } + + public boolean isMRShuffle() { + return (edgeType & MR_SHUFFLE_SORT) != 0; } public void setShuffleSort() { edgeType |= SHUFFLE_SORT; } - + + public boolean isShuffleSort() { + return (edgeType & SHUFFLE_SORT) != 0; + } + public long getEdgeType() { return edgeType; } @@ -77,11 +86,20 @@ public String getShuffleType() { sb.append("GROUP"); } + if (isMRShuffle()) { + if (sb.length() != 0) { + sb.append(" "); + } + sb.append("PARTITION-LEVEL SORT"); + return sb.toString(); + } + if (isShuffleSort()) { if (sb.length() != 0) { sb.append(" "); } sb.append("SORT"); + return sb.toString(); } return sb.toString(); diff --git ql/src/test/queries/clientpositive/parallel_join0.q ql/src/test/queries/clientpositive/parallel_join0.q new file mode 100644 index 0000000..5180947 --- /dev/null +++ ql/src/test/queries/clientpositive/parallel_join0.q @@ -0,0 +1,27 @@ +set mapreduce.job.reduces=4; + +-- SORT_BEFORE_DIFF + +EXPLAIN +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2; + +EXPLAIN FORMATTED +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2; + +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2; + diff --git ql/src/test/queries/clientpositive/parallel_join1.q ql/src/test/queries/clientpositive/parallel_join1.q new file mode 100644 index 0000000..3e29fbb --- /dev/null +++ ql/src/test/queries/clientpositive/parallel_join1.q @@ -0,0 +1,15 @@ +set mapreduce.job.reduces=4; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +-- SORT_BEFORE_DIFF + +CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value; + +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value; + +SELECT dest_j1.* FROM dest_j1; diff --git ql/src/test/results/clientpositive/parallel_join0.q.out ql/src/test/results/clientpositive/parallel_join0.q.out new file mode 100644 index 0000000..9fdbe18 --- /dev/null +++ ql/src/test/results/clientpositive/parallel_join0.q.out @@ -0,0 +1,242 @@ +Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: -- SORT_BEFORE_DIFF + +EXPLAIN +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_BEFORE_DIFF + +EXPLAIN +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} {VALUE._col1} + 1 {VALUE._col0} {VALUE._col1} + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: ++++ + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: EXPLAIN FORMATTED +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FORMATTED +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +{"STAGE PLANS":{"Stage-2":{"Map Reduce":{"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"File Output Operator":{"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","compressed:":"false","table:":{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE"}},"Map Operator Tree:":[{"TableScan":{"children":{"Reduce Output Operator":{"sort order:":"++++","Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)"}}}}]}},"Stage-1":{"Map Reduce":{"Reduce Operator Tree:":{"Join Operator":{"outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"Select Operator":{"expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"File Output Operator":{"compressed:":"false","table:":{"serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe","input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"}}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","condition map:":[{"":"Inner Join 0 to 1"}],"condition expressions:":{"1":"{VALUE._col0} {VALUE._col1}","0":"{VALUE._col0} {VALUE._col1}"}}},"Map Operator Tree:":[{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE"}},{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE"}}]}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}},"STAGE DEPENDENCIES":{"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}}} +Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 4 val_4 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 8 val_8 +0 val_0 9 val_9 +2 val_2 0 val_0 +2 val_2 5 val_5 +4 val_4 0 val_0 +4 val_4 2 val_2 +4 val_4 4 val_4 +4 val_4 5 val_5 +4 val_4 5 val_5 +4 val_4 9 val_9 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 2 val_2 +5 val_5 2 val_2 +5 val_5 4 val_4 +5 val_5 5 val_5 +5 val_5 8 val_8 +5 val_5 9 val_9 +5 val_5 9 val_9 +8 val_8 0 val_0 +8 val_8 4 val_4 +8 val_8 5 val_5 +8 val_8 8 val_8 +9 val_9 5 val_5 +9 val_9 5 val_5 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 4 val_4 +0 val_0 4 val_4 +0 val_0 5 val_5 +0 val_0 8 val_8 +2 val_2 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 4 val_4 +5 val_5 4 val_4 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +8 val_8 0 val_0 +8 val_8 2 val_2 +8 val_8 5 val_5 +9 val_9 2 val_2 +9 val_9 9 val_9 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 2 val_2 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 9 val_9 +2 val_2 0 val_0 +2 val_2 4 val_4 +2 val_2 5 val_5 +2 val_2 5 val_5 +2 val_2 8 val_8 +4 val_4 0 val_0 +4 val_4 0 val_0 +4 val_4 5 val_5 +4 val_4 8 val_8 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 8 val_8 +8 val_8 0 val_0 +8 val_8 5 val_5 +9 val_9 0 val_0 +9 val_9 0 val_0 +9 val_9 5 val_5 +9 val_9 8 val_8 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 2 val_2 +0 val_0 2 val_2 +0 val_0 5 val_5 +0 val_0 8 val_8 +0 val_0 9 val_9 +2 val_2 2 val_2 +2 val_2 9 val_9 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 2 val_2 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 8 val_8 +5 val_5 9 val_9 +8 val_8 9 val_9 +9 val_9 0 val_0 +9 val_9 4 val_4 diff --git ql/src/test/results/clientpositive/parallel_join1.q.out ql/src/test/results/clientpositive/parallel_join1.q.out new file mode 100644 index 0000000..81604b5 --- /dev/null +++ ql/src/test/results/clientpositive/parallel_join1.q.out @@ -0,0 +1,1135 @@ +PREHOOK: query: -- SORT_BEFORE_DIFF + +CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: -- SORT_BEFORE_DIFF + +CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col0} + outputColumnNames: _col0, _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest_j1.* FROM dest_j1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest_j1.* FROM dest_j1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +103 val_103 +103 val_103 +103 val_103 +103 val_103 +11 val_11 +114 val_114 +118 val_118 +118 val_118 +118 val_118 +118 val_118 +125 val_125 +125 val_125 +125 val_125 +125 val_125 +129 val_129 +129 val_129 +129 val_129 +129 val_129 +136 val_136 +143 val_143 +15 val_15 +15 val_15 +15 val_15 +15 val_15 +150 val_150 +158 val_158 +165 val_165 +165 val_165 +165 val_165 +165 val_165 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +172 val_172 +172 val_172 +172 val_172 +172 val_172 +176 val_176 +176 val_176 +176 val_176 +176 val_176 +183 val_183 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +19 val_19 +190 val_190 +194 val_194 +202 val_202 +213 val_213 +213 val_213 +213 val_213 +213 val_213 +217 val_217 +217 val_217 +217 val_217 +217 val_217 +224 val_224 +224 val_224 +224 val_224 +224 val_224 +228 val_228 +235 val_235 +239 val_239 +239 val_239 +239 val_239 +239 val_239 +242 val_242 +242 val_242 +242 val_242 +242 val_242 +257 val_257 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +260 val_260 +275 val_275 +282 val_282 +282 val_282 +282 val_282 +282 val_282 +286 val_286 +305 val_305 +309 val_309 +309 val_309 +309 val_309 +309 val_309 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +323 val_323 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +33 val_33 +338 val_338 +341 val_341 +345 val_345 +356 val_356 +367 val_367 +367 val_367 +367 val_367 +367 val_367 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +374 val_374 +378 val_378 +389 val_389 +392 val_392 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +4 val_4 +400 val_400 +404 val_404 +404 val_404 +404 val_404 +404 val_404 +411 val_411 +419 val_419 +437 val_437 +44 val_44 +444 val_444 +448 val_448 +455 val_455 +459 val_459 +459 val_459 +459 val_459 +459 val_459 +462 val_462 +462 val_462 +462 val_462 +462 val_462 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +477 val_477 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +484 val_484 +491 val_491 +495 val_495 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +66 val_66 +77 val_77 +8 val_8 +80 val_80 +84 val_84 +84 val_84 +84 val_84 +84 val_84 +95 val_95 +95 val_95 +95 val_95 +95 val_95 +100 val_100 +100 val_100 +100 val_100 +100 val_100 +104 val_104 +104 val_104 +104 val_104 +104 val_104 +111 val_111 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +126 val_126 +133 val_133 +137 val_137 +137 val_137 +137 val_137 +137 val_137 +155 val_155 +162 val_162 +166 val_166 +177 val_177 +180 val_180 +191 val_191 +191 val_191 +191 val_191 +191 val_191 +195 val_195 +195 val_195 +195 val_195 +195 val_195 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +203 val_203 +203 val_203 +203 val_203 +203 val_203 +207 val_207 +207 val_207 +207 val_207 +207 val_207 +214 val_214 +218 val_218 +221 val_221 +221 val_221 +221 val_221 +221 val_221 +229 val_229 +229 val_229 +229 val_229 +229 val_229 +247 val_247 +258 val_258 +265 val_265 +265 val_265 +265 val_265 +265 val_265 +27 val_27 +272 val_272 +272 val_272 +272 val_272 +272 val_272 +283 val_283 +287 val_287 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +30 val_30 +302 val_302 +306 val_306 +317 val_317 +317 val_317 +317 val_317 +317 val_317 +331 val_331 +331 val_331 +331 val_331 +331 val_331 +335 val_335 +339 val_339 +34 val_34 +342 val_342 +342 val_342 +342 val_342 +342 val_342 +353 val_353 +353 val_353 +353 val_353 +353 val_353 +360 val_360 +364 val_364 +368 val_368 +375 val_375 +379 val_379 +382 val_382 +382 val_382 +382 val_382 +382 val_382 +386 val_386 +393 val_393 +397 val_397 +397 val_397 +397 val_397 +397 val_397 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +41 val_41 +427 val_427 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +449 val_449 +452 val_452 +463 val_463 +463 val_463 +463 val_463 +463 val_463 +467 val_467 +470 val_470 +478 val_478 +478 val_478 +478 val_478 +478 val_478 +481 val_481 +485 val_485 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +492 val_492 +492 val_492 +492 val_492 +492 val_492 +496 val_496 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +74 val_74 +78 val_78 +85 val_85 +9 val_9 +92 val_92 +96 val_96 +105 val_105 +116 val_116 +134 val_134 +134 val_134 +134 val_134 +134 val_134 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +145 val_145 +149 val_149 +149 val_149 +149 val_149 +149 val_149 +152 val_152 +152 val_152 +152 val_152 +152 val_152 +156 val_156 +163 val_163 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +17 val_17 +170 val_170 +174 val_174 +174 val_174 +174 val_174 +174 val_174 +178 val_178 +181 val_181 +189 val_189 +192 val_192 +196 val_196 +2 val_2 +20 val_20 +200 val_200 +200 val_200 +200 val_200 +200 val_200 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +219 val_219 +219 val_219 +219 val_219 +219 val_219 +222 val_222 +226 val_226 +233 val_233 +233 val_233 +233 val_233 +233 val_233 +237 val_237 +237 val_237 +237 val_237 +237 val_237 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +244 val_244 +248 val_248 +255 val_255 +255 val_255 +255 val_255 +255 val_255 +262 val_262 +266 val_266 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +28 val_28 +280 val_280 +280 val_280 +280 val_280 +280 val_280 +284 val_284 +288 val_288 +288 val_288 +288 val_288 +288 val_288 +291 val_291 +307 val_307 +307 val_307 +307 val_307 +307 val_307 +310 val_310 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +321 val_321 +321 val_321 +321 val_321 +321 val_321 +325 val_325 +325 val_325 +325 val_325 +325 val_325 +332 val_332 +336 val_336 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +365 val_365 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +394 val_394 +402 val_402 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +413 val_413 +413 val_413 +413 val_413 +413 val_413 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +424 val_424 +424 val_424 +424 val_424 +424 val_424 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +435 val_435 +439 val_439 +439 val_439 +439 val_439 +439 val_439 +446 val_446 +453 val_453 +457 val_457 +460 val_460 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +475 val_475 +479 val_479 +482 val_482 +493 val_493 +497 val_497 +53 val_53 +57 val_57 +64 val_64 +82 val_82 +86 val_86 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +10 val_10 +113 val_113 +113 val_113 +113 val_113 +113 val_113 +120 val_120 +120 val_120 +120 val_120 +120 val_120 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +131 val_131 +146 val_146 +146 val_146 +146 val_146 +146 val_146 +153 val_153 +157 val_157 +160 val_160 +164 val_164 +164 val_164 +164 val_164 +164 val_164 +168 val_168 +175 val_175 +175 val_175 +175 val_175 +175 val_175 +179 val_179 +179 val_179 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +18 val_18 +18 val_18 +186 val_186 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +197 val_197 +197 val_197 +197 val_197 +197 val_197 +201 val_201 +205 val_205 +205 val_205 +205 val_205 +205 val_205 +209 val_209 +209 val_209 +209 val_209 +209 val_209 +216 val_216 +216 val_216 +216 val_216 +216 val_216 +223 val_223 +223 val_223 +223 val_223 +223 val_223 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +238 val_238 +238 val_238 +238 val_238 +238 val_238 +241 val_241 +249 val_249 +252 val_252 +256 val_256 +256 val_256 +256 val_256 +256 val_256 +263 val_263 +274 val_274 +278 val_278 +278 val_278 +278 val_278 +278 val_278 +281 val_281 +281 val_281 +281 val_281 +281 val_281 +285 val_285 +289 val_289 +292 val_292 +296 val_296 +308 val_308 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +322 val_322 +322 val_322 +322 val_322 +322 val_322 +333 val_333 +333 val_333 +333 val_333 +333 val_333 +344 val_344 +344 val_344 +344 val_344 +344 val_344 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +351 val_351 +362 val_362 +366 val_366 +373 val_373 +377 val_377 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +395 val_395 +395 val_395 +395 val_395 +395 val_395 +399 val_399 +399 val_399 +399 val_399 +399 val_399 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +407 val_407 +414 val_414 +414 val_414 +414 val_414 +414 val_414 +418 val_418 +421 val_421 +429 val_429 +429 val_429 +429 val_429 +429 val_429 +43 val_43 +432 val_432 +436 val_436 +443 val_443 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +458 val_458 +458 val_458 +458 val_458 +458 val_458 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +47 val_47 +472 val_472 +483 val_483 +487 val_487 +490 val_490 +494 val_494 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +54 val_54 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +65 val_65 +69 val_69 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +83 val_83 +83 val_83 +83 val_83 +83 val_83 +87 val_87 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +98 val_98 +98 val_98 +98 val_98 +98 val_98 diff --git ql/src/test/results/clientpositive/spark/char_join1.q.out ql/src/test/results/clientpositive/spark/char_join1.q.out new file mode 100644 index 0000000..48e6ad8 --- /dev/null +++ ql/src/test/results/clientpositive/spark/char_join1.q.out @@ -0,0 +1,153 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +drop table char_join1_ch1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +drop table char_join1_ch1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table char_join1_ch2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table char_join1_ch2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table char_join1_str +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table char_join1_str +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table char_join1_ch1 ( + c1 int, + c2 char(10) +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@char_join1_ch1 +POSTHOOK: query: create table char_join1_ch1 ( + c1 int, + c2 char(10) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@char_join1_ch1 +PREHOOK: query: create table char_join1_ch2 ( + c1 int, + c2 char(20) +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@char_join1_ch2 +POSTHOOK: query: create table char_join1_ch2 ( + c1 int, + c2 char(20) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@char_join1_ch2 +PREHOOK: query: create table char_join1_str ( + c1 int, + c2 string +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@char_join1_str +POSTHOOK: query: create table char_join1_str ( + c1 int, + c2 string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@char_join1_str +PREHOOK: query: load data local inpath '../../data/files/vc1.txt' into table char_join1_ch1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@char_join1_ch1 +POSTHOOK: query: load data local inpath '../../data/files/vc1.txt' into table char_join1_ch1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@char_join1_ch1 +PREHOOK: query: load data local inpath '../../data/files/vc1.txt' into table char_join1_ch2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@char_join1_ch2 +POSTHOOK: query: load data local inpath '../../data/files/vc1.txt' into table char_join1_ch2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@char_join1_ch2 +PREHOOK: query: load data local inpath '../../data/files/vc1.txt' into table char_join1_str +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@char_join1_str +POSTHOOK: query: load data local inpath '../../data/files/vc1.txt' into table char_join1_str +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@char_join1_str +PREHOOK: query: -- Join char with same length char +select * from char_join1_ch1 a join char_join1_ch1 b on (a.c2 = b.c2) +PREHOOK: type: QUERY +PREHOOK: Input: default@char_join1_ch1 +#### A masked pattern was here #### +POSTHOOK: query: -- Join char with same length char +select * from char_join1_ch1 a join char_join1_ch1 b on (a.c2 = b.c2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@char_join1_ch1 +#### A masked pattern was here #### +1 abc 1 abc +1 abc 2 abc +2 abc 1 abc +2 abc 2 abc +3 abc 3 abc +PREHOOK: query: -- Join char with different length char +select * from char_join1_ch1 a join char_join1_ch2 b on (a.c2 = b.c2) +PREHOOK: type: QUERY +PREHOOK: Input: default@char_join1_ch1 +PREHOOK: Input: default@char_join1_ch2 +#### A masked pattern was here #### +POSTHOOK: query: -- Join char with different length char +select * from char_join1_ch1 a join char_join1_ch2 b on (a.c2 = b.c2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@char_join1_ch1 +POSTHOOK: Input: default@char_join1_ch2 +#### A masked pattern was here #### +1 abc 1 abc +1 abc 2 abc +2 abc 1 abc +2 abc 2 abc +3 abc 3 abc +PREHOOK: query: -- Join char with string +select * from char_join1_ch1 a join char_join1_str b on (a.c2 = b.c2) +PREHOOK: type: QUERY +PREHOOK: Input: default@char_join1_ch1 +PREHOOK: Input: default@char_join1_str +#### A masked pattern was here #### +POSTHOOK: query: -- Join char with string +select * from char_join1_ch1 a join char_join1_str b on (a.c2 = b.c2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@char_join1_ch1 +POSTHOOK: Input: default@char_join1_str +#### A masked pattern was here #### +1 abc 1 abc +2 abc 1 abc +3 abc 3 abc +PREHOOK: query: drop table char_join1_ch1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@char_join1_ch1 +PREHOOK: Output: default@char_join1_ch1 +POSTHOOK: query: drop table char_join1_ch1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@char_join1_ch1 +POSTHOOK: Output: default@char_join1_ch1 +PREHOOK: query: drop table char_join1_ch2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@char_join1_ch2 +PREHOOK: Output: default@char_join1_ch2 +POSTHOOK: query: drop table char_join1_ch2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@char_join1_ch2 +POSTHOOK: Output: default@char_join1_ch2 +PREHOOK: query: drop table char_join1_str +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@char_join1_str +PREHOOK: Output: default@char_join1_str +POSTHOOK: query: drop table char_join1_str +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@char_join1_str +POSTHOOK: Output: default@char_join1_str diff --git ql/src/test/results/clientpositive/spark/column_access_stats.q.out ql/src/test/results/clientpositive/spark/column_access_stats.q.out index 56b763e..6c1c665 100644 --- ql/src/test/results/clientpositive/spark/column_access_stats.q.out +++ ql/src/test/results/clientpositive/spark/column_access_stats.q.out @@ -411,7 +411,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -534,7 +534,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -624,7 +624,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -736,8 +736,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Reducer 4 (GROUP SORT) - Reducer 4 <- Map 3 (GROUP SORT), Map 5 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 4 (GROUP PARTITION-LEVEL SORT) + Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT), Map 5 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/groupby_position.q.out ql/src/test/results/clientpositive/spark/groupby_position.q.out index bef99c9..924e02b 100644 --- ql/src/test/results/clientpositive/spark/groupby_position.q.out +++ ql/src/test/results/clientpositive/spark/groupby_position.q.out @@ -621,7 +621,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP SORT) - Reducer 3 <- Reducer 2 (GROUP SORT), Reducer 6 (GROUP SORT) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT), Reducer 6 (GROUP PARTITION-LEVEL SORT) Reducer 4 <- Reducer 3 (GROUP SORT) Reducer 6 <- Map 5 (GROUP) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 5b5495d..c256b90 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -2784,7 +2784,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -3168,7 +3168,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Reducer 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 4 (GROUP PARTITION-LEVEL SORT) Reducer 4 <- Map 3 (GROUP) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index 318fbbc..3d2cebe 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -2874,7 +2874,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -3258,7 +3258,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Reducer 5 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 5 (GROUP PARTITION-LEVEL SORT) Reducer 4 <- Map 3 (GROUP SORT) Reducer 5 <- Reducer 4 (GROUP) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/innerjoin.q.out ql/src/test/results/clientpositive/spark/innerjoin.q.out index acac2b9..0db53de 100644 --- ql/src/test/results/clientpositive/spark/innerjoin.q.out +++ ql/src/test/results/clientpositive/spark/innerjoin.q.out @@ -28,7 +28,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -1194,7 +1194,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join0.q.out ql/src/test/results/clientpositive/spark/join0.q.out index 913f57a..e0df0ad 100644 --- ql/src/test/results/clientpositive/spark/join0.q.out +++ ql/src/test/results/clientpositive/spark/join0.q.out @@ -22,7 +22,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP), Map 4 (GROUP) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (GROUP SORT) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/join1.q.out ql/src/test/results/clientpositive/spark/join1.q.out index 9db644b..517a983 100644 --- ql/src/test/results/clientpositive/spark/join1.q.out +++ ql/src/test/results/clientpositive/spark/join1.q.out @@ -24,7 +24,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join10.q.out ql/src/test/results/clientpositive/spark/join10.q.out index 5122c56..2184033 100644 --- ql/src/test/results/clientpositive/spark/join10.q.out +++ ql/src/test/results/clientpositive/spark/join10.q.out @@ -20,7 +20,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join11.q.out ql/src/test/results/clientpositive/spark/join11.q.out index f4a080f..a335787 100644 --- ql/src/test/results/clientpositive/spark/join11.q.out +++ ql/src/test/results/clientpositive/spark/join11.q.out @@ -22,7 +22,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join12.q.out ql/src/test/results/clientpositive/spark/join12.q.out index 1b5992f..40a7641 100644 --- ql/src/test/results/clientpositive/spark/join12.q.out +++ ql/src/test/results/clientpositive/spark/join12.q.out @@ -28,7 +28,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join13.q.out ql/src/test/results/clientpositive/spark/join13.q.out index c64bdb3..f247fad 100644 --- ql/src/test/results/clientpositive/spark/join13.q.out +++ ql/src/test/results/clientpositive/spark/join13.q.out @@ -28,8 +28,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 4 (GROUP SORT) - Reducer 3 <- Map 5 (GROUP SORT), Reducer 2 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) + Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT), Reducer 2 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join14.q.out ql/src/test/results/clientpositive/spark/join14.q.out index 9dcc6c8..fe40867 100644 --- ql/src/test/results/clientpositive/spark/join14.q.out +++ ql/src/test/results/clientpositive/spark/join14.q.out @@ -28,7 +28,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join15.q.out ql/src/test/results/clientpositive/spark/join15.q.out index ca7b5c5..3128191 100644 --- ql/src/test/results/clientpositive/spark/join15.q.out +++ ql/src/test/results/clientpositive/spark/join15.q.out @@ -12,7 +12,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (GROUP SORT) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/join16.q.out ql/src/test/results/clientpositive/spark/join16.q.out index 3a57bf5..c78704d 100644 --- ql/src/test/results/clientpositive/spark/join16.q.out +++ ql/src/test/results/clientpositive/spark/join16.q.out @@ -10,7 +10,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join17.q.out ql/src/test/results/clientpositive/spark/join17.q.out index 7c6d9ff..f491b7e 100644 --- ql/src/test/results/clientpositive/spark/join17.q.out +++ ql/src/test/results/clientpositive/spark/join17.q.out @@ -62,7 +62,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join18.q.out ql/src/test/results/clientpositive/spark/join18.q.out index 3278dde..1e22ff5 100644 --- ql/src/test/results/clientpositive/spark/join18.q.out +++ ql/src/test/results/clientpositive/spark/join18.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP SORT) - Reducer 3 <- Reducer 2 (GROUP SORT), Reducer 5 (GROUP SORT) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT), Reducer 5 (GROUP PARTITION-LEVEL SORT) Reducer 5 <- Map 4 (GROUP) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/join19.q.out ql/src/test/results/clientpositive/spark/join19.q.out index 87606fd..ae00709 100644 --- ql/src/test/results/clientpositive/spark/join19.q.out +++ ql/src/test/results/clientpositive/spark/join19.q.out @@ -126,9 +126,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Reducer 5 (GROUP SORT) - Reducer 4 <- Map 3 (GROUP SORT), Map 6 (GROUP SORT), Map 8 (GROUP SORT) - Reducer 5 <- Map 7 (GROUP SORT), Map 9 (GROUP SORT), Reducer 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 5 (GROUP PARTITION-LEVEL SORT) + Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT), Map 6 (GROUP PARTITION-LEVEL SORT), Map 8 (GROUP PARTITION-LEVEL SORT) + Reducer 5 <- Map 7 (GROUP PARTITION-LEVEL SORT), Map 9 (GROUP PARTITION-LEVEL SORT), Reducer 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join2.q.out ql/src/test/results/clientpositive/spark/join2.q.out index 0c3880b..dcb4338 100644 --- ql/src/test/results/clientpositive/spark/join2.q.out +++ ql/src/test/results/clientpositive/spark/join2.q.out @@ -24,8 +24,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 5 (GROUP SORT) - Reducer 3 <- Map 4 (GROUP SORT), Reducer 2 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 5 (GROUP PARTITION-LEVEL SORT) + Reducer 3 <- Map 4 (GROUP PARTITION-LEVEL SORT), Reducer 2 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join20.q.out ql/src/test/results/clientpositive/spark/join20.q.out index 56b4bed..2e5db05 100644 --- ql/src/test/results/clientpositive/spark/join20.q.out +++ ql/src/test/results/clientpositive/spark/join20.q.out @@ -14,7 +14,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 4 (GROUP SORT), Map 5 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT), Map 5 (GROUP PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (GROUP SORT) #### A masked pattern was here #### Vertices: @@ -675,7 +675,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 4 (GROUP SORT), Map 5 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT), Map 5 (GROUP PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (GROUP SORT) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/join21.q.out ql/src/test/results/clientpositive/spark/join21.q.out index 0e08bf8..47e2ee0 100644 --- ql/src/test/results/clientpositive/spark/join21.q.out +++ ql/src/test/results/clientpositive/spark/join21.q.out @@ -12,7 +12,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 4 (GROUP SORT), Map 5 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT), Map 5 (GROUP PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (GROUP SORT) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/join22.q.out ql/src/test/results/clientpositive/spark/join22.q.out index 1c8ab7c..7422d2f 100644 --- ql/src/test/results/clientpositive/spark/join22.q.out +++ ql/src/test/results/clientpositive/spark/join22.q.out @@ -12,8 +12,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 4 (GROUP SORT) - Reducer 3 <- Map 5 (GROUP SORT), Reducer 2 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) + Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT), Reducer 2 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join23.q.out ql/src/test/results/clientpositive/spark/join23.q.out index ecd8371..08ee10a 100644 --- ql/src/test/results/clientpositive/spark/join23.q.out +++ ql/src/test/results/clientpositive/spark/join23.q.out @@ -12,7 +12,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP), Map 4 (GROUP) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (GROUP SORT) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/join25.q.out ql/src/test/results/clientpositive/spark/join25.q.out index 71df358..b4459d3 100644 --- ql/src/test/results/clientpositive/spark/join25.q.out +++ ql/src/test/results/clientpositive/spark/join25.q.out @@ -30,7 +30,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join26.q.out ql/src/test/results/clientpositive/spark/join26.q.out index 06246a4..87f4c39 100644 --- ql/src/test/results/clientpositive/spark/join26.q.out +++ ql/src/test/results/clientpositive/spark/join26.q.out @@ -111,7 +111,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join27.q.out ql/src/test/results/clientpositive/spark/join27.q.out index 8cbe599..7e06b85 100644 --- ql/src/test/results/clientpositive/spark/join27.q.out +++ ql/src/test/results/clientpositive/spark/join27.q.out @@ -30,7 +30,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join3.q.out ql/src/test/results/clientpositive/spark/join3.q.out index 2f47a21..b41ebdd 100644 --- ql/src/test/results/clientpositive/spark/join3.q.out +++ ql/src/test/results/clientpositive/spark/join3.q.out @@ -24,7 +24,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join4.q.out ql/src/test/results/clientpositive/spark/join4.q.out index 48ea655..6512265 100644 --- ql/src/test/results/clientpositive/spark/join4.q.out +++ ql/src/test/results/clientpositive/spark/join4.q.out @@ -46,7 +46,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join5.q.out ql/src/test/results/clientpositive/spark/join5.q.out index d1130fe..fbf2ed4 100644 --- ql/src/test/results/clientpositive/spark/join5.q.out +++ ql/src/test/results/clientpositive/spark/join5.q.out @@ -46,7 +46,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join6.q.out ql/src/test/results/clientpositive/spark/join6.q.out index bfbe240..b95e0a4 100644 --- ql/src/test/results/clientpositive/spark/join6.q.out +++ ql/src/test/results/clientpositive/spark/join6.q.out @@ -46,7 +46,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join7.q.out ql/src/test/results/clientpositive/spark/join7.q.out index 1f5a4cc..7bd3bdc 100644 --- ql/src/test/results/clientpositive/spark/join7.q.out +++ ql/src/test/results/clientpositive/spark/join7.q.out @@ -56,7 +56,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join8.q.out ql/src/test/results/clientpositive/spark/join8.q.out index 70782cc..8d97aca 100644 --- ql/src/test/results/clientpositive/spark/join8.q.out +++ ql/src/test/results/clientpositive/spark/join8.q.out @@ -46,7 +46,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join9.q.out ql/src/test/results/clientpositive/spark/join9.q.out index f0c4172..05d68b7 100644 --- ql/src/test/results/clientpositive/spark/join9.q.out +++ ql/src/test/results/clientpositive/spark/join9.q.out @@ -78,7 +78,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/join_nullsafe.q.out ql/src/test/results/clientpositive/spark/join_nullsafe.q.out index 48d5d76..44d7f46 100644 --- ql/src/test/results/clientpositive/spark/join_nullsafe.q.out +++ ql/src/test/results/clientpositive/spark/join_nullsafe.q.out @@ -32,7 +32,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -117,7 +117,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -214,7 +214,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -330,7 +330,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -425,7 +425,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -1566,7 +1566,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/limit_pushdown.q.out ql/src/test/results/clientpositive/spark/limit_pushdown.q.out index d088c8a..ec8a377 100644 --- ql/src/test/results/clientpositive/spark/limit_pushdown.q.out +++ ql/src/test/results/clientpositive/spark/limit_pushdown.q.out @@ -812,7 +812,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP) - Reducer 3 <- Reducer 2 (GROUP SORT), Reducer 6 (GROUP SORT) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT), Reducer 6 (GROUP PARTITION-LEVEL SORT) Reducer 5 <- Map 4 (GROUP) Reducer 6 <- Reducer 5 (GROUP) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out index 33c470e..0254170 100644 --- ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out +++ ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out @@ -128,7 +128,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Reducer 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 4 (GROUP PARTITION-LEVEL SORT) Reducer 4 <- Map 3 (GROUP) #### A masked pattern was here #### Vertices: @@ -1005,7 +1005,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP), Reducer 4 (GROUP) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 4 (GROUP PARTITION-LEVEL SORT) Reducer 4 <- Map 3 (GROUP) #### A masked pattern was here #### Vertices: @@ -1442,7 +1442,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 5 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 5 (GROUP PARTITION-LEVEL SORT) Union 3 <- Map 4 (NONE), Reducer 2 (NONE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/spark/parallel_join0.q.out ql/src/test/results/clientpositive/spark/parallel_join0.q.out new file mode 100644 index 0000000..3149825 --- /dev/null +++ ql/src/test/results/clientpositive/spark/parallel_join0.q.out @@ -0,0 +1,237 @@ +PREHOOK: query: -- SORT_BEFORE_DIFF + +EXPLAIN +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_BEFORE_DIFF + +EXPLAIN +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) + Reducer 3 <- Reducer 2 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} {VALUE._col1} + 1 {VALUE._col0} {VALUE._col1} + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: ++++ + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN FORMATTED +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FORMATTED +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +#### A masked pattern was here #### +PREHOOK: query: SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 val_5 9 val_9 +5 val_5 9 val_9 +5 val_5 9 val_9 +8 val_8 0 val_0 +8 val_8 0 val_0 +8 val_8 0 val_0 +8 val_8 2 val_2 +8 val_8 4 val_4 +8 val_8 5 val_5 +8 val_8 5 val_5 +8 val_8 5 val_5 +8 val_8 8 val_8 +8 val_8 9 val_9 +9 val_9 0 val_0 +9 val_9 0 val_0 +9 val_9 0 val_0 +9 val_9 2 val_2 +9 val_9 4 val_4 +9 val_9 5 val_5 +9 val_9 5 val_5 +9 val_9 5 val_5 +9 val_9 8 val_8 +9 val_9 9 val_9 +5 val_5 2 val_2 +5 val_5 2 val_2 +5 val_5 2 val_2 +5 val_5 4 val_4 +5 val_5 4 val_4 +5 val_5 4 val_4 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 8 val_8 +5 val_5 8 val_8 +5 val_5 8 val_8 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 2 val_2 +0 val_0 2 val_2 +0 val_0 2 val_2 +0 val_0 4 val_4 +0 val_0 4 val_4 +0 val_0 4 val_4 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 5 val_5 +0 val_0 8 val_8 +0 val_0 8 val_8 +0 val_0 8 val_8 +0 val_0 9 val_9 +0 val_0 9 val_9 +0 val_0 9 val_9 +2 val_2 0 val_0 +2 val_2 0 val_0 +2 val_2 0 val_0 +2 val_2 2 val_2 +2 val_2 4 val_4 +2 val_2 5 val_5 +2 val_2 5 val_5 +2 val_2 5 val_5 +2 val_2 8 val_8 +2 val_2 9 val_9 +4 val_4 0 val_0 +4 val_4 0 val_0 +4 val_4 0 val_0 +4 val_4 2 val_2 +4 val_4 4 val_4 +4 val_4 5 val_5 +4 val_4 5 val_5 +4 val_4 5 val_5 +4 val_4 8 val_8 +4 val_4 9 val_9 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 +5 val_5 0 val_0 diff --git ql/src/test/results/clientpositive/spark/parallel_join1.q.out ql/src/test/results/clientpositive/spark/parallel_join1.q.out new file mode 100644 index 0000000..7cd0af6 --- /dev/null +++ ql/src/test/results/clientpositive/spark/parallel_join1.q.out @@ -0,0 +1,1148 @@ +PREHOOK: query: -- SORT_BEFORE_DIFF + +CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: -- SORT_BEFORE_DIFF + +CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col0} + outputColumnNames: _col0, _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest_j1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest_j1.* FROM dest_j1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest_j1.* FROM dest_j1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +#### A masked pattern was here #### +10 val_10 +113 val_113 +113 val_113 +113 val_113 +113 val_113 +120 val_120 +120 val_120 +120 val_120 +120 val_120 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +131 val_131 +146 val_146 +146 val_146 +146 val_146 +146 val_146 +153 val_153 +157 val_157 +160 val_160 +164 val_164 +164 val_164 +164 val_164 +164 val_164 +168 val_168 +175 val_175 +175 val_175 +175 val_175 +175 val_175 +179 val_179 +179 val_179 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +18 val_18 +18 val_18 +186 val_186 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +197 val_197 +197 val_197 +197 val_197 +197 val_197 +201 val_201 +205 val_205 +205 val_205 +205 val_205 +205 val_205 +209 val_209 +209 val_209 +209 val_209 +209 val_209 +216 val_216 +216 val_216 +216 val_216 +216 val_216 +223 val_223 +223 val_223 +223 val_223 +223 val_223 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +238 val_238 +238 val_238 +238 val_238 +238 val_238 +241 val_241 +249 val_249 +252 val_252 +256 val_256 +256 val_256 +256 val_256 +256 val_256 +263 val_263 +274 val_274 +278 val_278 +278 val_278 +278 val_278 +278 val_278 +281 val_281 +281 val_281 +281 val_281 +281 val_281 +285 val_285 +289 val_289 +292 val_292 +296 val_296 +308 val_308 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +322 val_322 +322 val_322 +322 val_322 +322 val_322 +333 val_333 +333 val_333 +333 val_333 +333 val_333 +344 val_344 +344 val_344 +344 val_344 +344 val_344 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +351 val_351 +362 val_362 +366 val_366 +373 val_373 +377 val_377 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +395 val_395 +395 val_395 +395 val_395 +395 val_395 +399 val_399 +399 val_399 +399 val_399 +399 val_399 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +407 val_407 +414 val_414 +414 val_414 +414 val_414 +414 val_414 +418 val_418 +421 val_421 +429 val_429 +429 val_429 +429 val_429 +429 val_429 +43 val_43 +432 val_432 +436 val_436 +443 val_443 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +458 val_458 +458 val_458 +458 val_458 +458 val_458 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +47 val_47 +472 val_472 +483 val_483 +487 val_487 +490 val_490 +494 val_494 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +54 val_54 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +65 val_65 +69 val_69 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +83 val_83 +83 val_83 +83 val_83 +83 val_83 +87 val_87 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +98 val_98 +98 val_98 +98 val_98 +98 val_98 +100 val_100 +100 val_100 +100 val_100 +100 val_100 +104 val_104 +104 val_104 +104 val_104 +104 val_104 +111 val_111 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +126 val_126 +133 val_133 +137 val_137 +137 val_137 +137 val_137 +137 val_137 +155 val_155 +162 val_162 +166 val_166 +177 val_177 +180 val_180 +191 val_191 +191 val_191 +191 val_191 +191 val_191 +195 val_195 +195 val_195 +195 val_195 +195 val_195 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +203 val_203 +203 val_203 +203 val_203 +203 val_203 +207 val_207 +207 val_207 +207 val_207 +207 val_207 +214 val_214 +218 val_218 +221 val_221 +221 val_221 +221 val_221 +221 val_221 +229 val_229 +229 val_229 +229 val_229 +229 val_229 +247 val_247 +258 val_258 +265 val_265 +265 val_265 +265 val_265 +265 val_265 +27 val_27 +272 val_272 +272 val_272 +272 val_272 +272 val_272 +283 val_283 +287 val_287 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +30 val_30 +302 val_302 +306 val_306 +317 val_317 +317 val_317 +317 val_317 +317 val_317 +331 val_331 +331 val_331 +331 val_331 +331 val_331 +335 val_335 +339 val_339 +34 val_34 +342 val_342 +342 val_342 +342 val_342 +342 val_342 +353 val_353 +353 val_353 +353 val_353 +353 val_353 +360 val_360 +364 val_364 +368 val_368 +375 val_375 +379 val_379 +382 val_382 +382 val_382 +382 val_382 +382 val_382 +386 val_386 +393 val_393 +397 val_397 +397 val_397 +397 val_397 +397 val_397 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +41 val_41 +427 val_427 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +449 val_449 +452 val_452 +463 val_463 +463 val_463 +463 val_463 +463 val_463 +467 val_467 +470 val_470 +478 val_478 +478 val_478 +478 val_478 +478 val_478 +481 val_481 +485 val_485 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +492 val_492 +492 val_492 +492 val_492 +492 val_492 +496 val_496 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +74 val_74 +78 val_78 +85 val_85 +9 val_9 +92 val_92 +96 val_96 +105 val_105 +116 val_116 +134 val_134 +134 val_134 +134 val_134 +134 val_134 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +145 val_145 +149 val_149 +149 val_149 +149 val_149 +149 val_149 +152 val_152 +152 val_152 +152 val_152 +152 val_152 +156 val_156 +163 val_163 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +17 val_17 +170 val_170 +174 val_174 +174 val_174 +174 val_174 +174 val_174 +178 val_178 +181 val_181 +189 val_189 +192 val_192 +196 val_196 +2 val_2 +20 val_20 +200 val_200 +200 val_200 +200 val_200 +200 val_200 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +219 val_219 +219 val_219 +219 val_219 +219 val_219 +222 val_222 +226 val_226 +233 val_233 +233 val_233 +233 val_233 +233 val_233 +237 val_237 +237 val_237 +237 val_237 +237 val_237 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +244 val_244 +248 val_248 +255 val_255 +255 val_255 +255 val_255 +255 val_255 +262 val_262 +266 val_266 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +28 val_28 +280 val_280 +280 val_280 +280 val_280 +280 val_280 +284 val_284 +288 val_288 +288 val_288 +288 val_288 +288 val_288 +291 val_291 +307 val_307 +307 val_307 +307 val_307 +307 val_307 +310 val_310 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +321 val_321 +321 val_321 +321 val_321 +321 val_321 +325 val_325 +325 val_325 +325 val_325 +325 val_325 +332 val_332 +336 val_336 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +365 val_365 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +394 val_394 +402 val_402 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +413 val_413 +413 val_413 +413 val_413 +413 val_413 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +424 val_424 +424 val_424 +424 val_424 +424 val_424 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +435 val_435 +439 val_439 +439 val_439 +439 val_439 +439 val_439 +446 val_446 +453 val_453 +457 val_457 +460 val_460 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +475 val_475 +479 val_479 +482 val_482 +493 val_493 +497 val_497 +53 val_53 +57 val_57 +64 val_64 +82 val_82 +86 val_86 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +103 val_103 +103 val_103 +103 val_103 +103 val_103 +11 val_11 +114 val_114 +118 val_118 +118 val_118 +118 val_118 +118 val_118 +125 val_125 +125 val_125 +125 val_125 +125 val_125 +129 val_129 +129 val_129 +129 val_129 +129 val_129 +136 val_136 +143 val_143 +15 val_15 +15 val_15 +15 val_15 +15 val_15 +150 val_150 +158 val_158 +165 val_165 +165 val_165 +165 val_165 +165 val_165 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +172 val_172 +172 val_172 +172 val_172 +172 val_172 +176 val_176 +176 val_176 +176 val_176 +176 val_176 +183 val_183 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +19 val_19 +190 val_190 +194 val_194 +202 val_202 +213 val_213 +213 val_213 +213 val_213 +213 val_213 +217 val_217 +217 val_217 +217 val_217 +217 val_217 +224 val_224 +224 val_224 +224 val_224 +224 val_224 +228 val_228 +235 val_235 +239 val_239 +239 val_239 +239 val_239 +239 val_239 +242 val_242 +242 val_242 +242 val_242 +242 val_242 +257 val_257 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +260 val_260 +275 val_275 +282 val_282 +282 val_282 +282 val_282 +282 val_282 +286 val_286 +305 val_305 +309 val_309 +309 val_309 +309 val_309 +309 val_309 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +323 val_323 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +33 val_33 +338 val_338 +341 val_341 +345 val_345 +356 val_356 +367 val_367 +367 val_367 +367 val_367 +367 val_367 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +374 val_374 +378 val_378 +389 val_389 +392 val_392 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +4 val_4 +400 val_400 +404 val_404 +404 val_404 +404 val_404 +404 val_404 +411 val_411 +419 val_419 +437 val_437 +44 val_44 +444 val_444 +448 val_448 +455 val_455 +459 val_459 +459 val_459 +459 val_459 +459 val_459 +462 val_462 +462 val_462 +462 val_462 +462 val_462 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +477 val_477 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +484 val_484 +491 val_491 +495 val_495 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +66 val_66 +77 val_77 +8 val_8 +80 val_80 +84 val_84 +84 val_84 +84 val_84 +84 val_84 +95 val_95 +95 val_95 +95 val_95 +95 val_95 diff --git ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out index 76b531e..a118823 100644 --- ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out +++ ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out @@ -55,7 +55,7 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 6 <- Map 5 (GROUP SORT), Map 7 (GROUP SORT) + Reducer 6 <- Map 5 (GROUP PARTITION-LEVEL SORT), Map 7 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 5 @@ -1361,7 +1361,7 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 6 <- Map 5 (GROUP SORT), Map 7 (GROUP SORT) + Reducer 6 <- Map 5 (GROUP PARTITION-LEVEL SORT), Map 7 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 5 diff --git ql/src/test/results/clientpositive/spark/sample8.q.out ql/src/test/results/clientpositive/spark/sample8.q.out index 365468b..d3b6da2 100644 --- ql/src/test/results/clientpositive/spark/sample8.q.out +++ ql/src/test/results/clientpositive/spark/sample8.q.out @@ -92,7 +92,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -808,7 +808,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 @@ -899,7 +899,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 3 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 3 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out index d38c554..a1ac731 100644 --- ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out +++ ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out @@ -87,8 +87,8 @@ STAGE PLANS: Spark Edges: Reducer 10 <- Map 9 (GROUP) - Reducer 11 <- Map 1 (GROUP), Reducer 10 (GROUP) - Reducer 7 <- Map 6 (GROUP SORT), Reducer 11 (GROUP SORT) + Reducer 11 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 10 (GROUP PARTITION-LEVEL SORT) + Reducer 7 <- Map 6 (GROUP PARTITION-LEVEL SORT), Reducer 11 (GROUP PARTITION-LEVEL SORT) Reducer 8 <- Reducer 7 (GROUP SORT) #### A masked pattern was here #### Vertices: @@ -242,7 +242,7 @@ STAGE PLANS: Stage: Stage-5 Spark Edges: - Reducer 5 <- Map 2 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 5 <- Map 2 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 2 @@ -549,8 +549,8 @@ STAGE PLANS: Spark Edges: Reducer 10 <- Map 9 (GROUP) - Reducer 11 <- Map 1 (GROUP), Reducer 10 (GROUP) - Reducer 7 <- Map 6 (GROUP SORT), Reducer 11 (GROUP SORT) + Reducer 11 <- Map 1 (GROUP PARTITION-LEVEL SORT), Reducer 10 (GROUP PARTITION-LEVEL SORT) + Reducer 7 <- Map 6 (GROUP PARTITION-LEVEL SORT), Reducer 11 (GROUP PARTITION-LEVEL SORT) Reducer 8 <- Reducer 7 (GROUP SORT) #### A masked pattern was here #### Vertices: @@ -704,7 +704,7 @@ STAGE PLANS: Stage: Stage-5 Spark Edges: - Reducer 5 <- Map 2 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 5 <- Map 2 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) #### A masked pattern was here #### Vertices: Map 2 diff --git ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out index bc4262b..776ce50 100644 --- ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out +++ ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out @@ -14,7 +14,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP SORT), Map 4 (GROUP SORT) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT), Map 4 (GROUP PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (GROUP) #### A masked pattern was here #### Vertices: