diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties
index f90b833..922efb8 100644
--- a/data/conf/hive-log4j.properties
+++ b/data/conf/hive-log4j.properties
@@ -42,7 +42,7 @@ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
# Pattern format: Date LogLevel LoggerName LogMessage
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
#
diff --git a/data/conf/spark/hive-site.xml b/data/conf/spark/hive-site.xml
index f3d7b40..59c4cd9 100644
--- a/data/conf/spark/hive-site.xml
+++ b/data/conf/spark/hive-site.xml
@@ -197,7 +197,12 @@
spark.master
- local[4]
+ local-cluster[2,2,1024]
+
+
+
+ spark.akka.logLifecycleEvents
+ true
diff --git a/itests/qtest-spark/.gitignore b/itests/qtest-spark/.gitignore
new file mode 100644
index 0000000..c2ed135
--- /dev/null
+++ b/itests/qtest-spark/.gitignore
@@ -0,0 +1 @@
+thirdparty/
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index a533e8d..fb121ff 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -339,6 +339,41 @@
maven-antrun-plugin
+ download-spark
+ generate-sources
+
+ run
+
+
+
+
+ set -e
+ set -x
+ /bin/pwd
+ BASE_DIR=./target
+ DOWNLOAD_DIR=./thirdparty
+ download() {
+ url=$1;
+ finalName=$2
+ tarName=$(basename $url)
+ rm -rf $BASE_DIR/$finalName
+ if [[ ! -f $DOWNLOAD_DIR/$tarName ]]
+ then
+ curl -Sso $DOWNLOAD_DIR/$tarName $url
+ fi
+ tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
+ mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName
+ }
+ mkdir -p $DOWNLOAD_DIR
+ download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark"
+
+
+
+
+
+
+
+
generate-tests-sources
generate-test-sources
diff --git a/pom.xml b/pom.xml
index cce2e5e..900dddb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -171,6 +171,7 @@
2.4.0
2.6.0
3.0.0
+ ${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark
@@ -836,6 +837,7 @@
true
${test.tmp.dir}
+ ${spark.home}
${test.tmp.dir}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
index 72108cd..c075c35 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
@@ -67,6 +67,9 @@ public int startMonitor() {
while (true) {
try {
JobExecutionStatus state = sparkJobStatus.getState();
+ if (LOG.isDebugEnabled()) {
+ console.printInfo("state = " + state);
+ }
if (state != null && state != JobExecutionStatus.UNKNOWN &&
(state != lastState || state == JobExecutionStatus.RUNNING)) {
lastState = state;
diff --git a/ql/src/test/queries/clientpositive/auto_join_without_localtask.q b/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
index f23e227..91aa963 100644
--- a/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
+++ b/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
@@ -5,14 +5,14 @@ set hive.auto.convert.join.use.nonstaged=true;
set hive.auto.convert.join.noconditionaltask.size=100;
explain
-select a.* from src a join src b on a.key=b.key limit 40;
+select a.* from src a join src b on a.key=b.key order by key, value limit 40;
-select a.* from src a join src b on a.key=b.key limit 40;
+select a.* from src a join src b on a.key=b.key order by key, value limit 40;
explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40;
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40;
set hive.auto.convert.join.noconditionaltask.size=100;
diff --git a/ql/src/test/queries/clientpositive/count.q b/ql/src/test/queries/clientpositive/count.q
index 74ae9e4..18721e5 100644
--- a/ql/src/test/queries/clientpositive/count.q
+++ b/ql/src/test/queries/clientpositive/count.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
create table abcd (a int, b int, c int, d int);
LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd;
diff --git a/ql/src/test/queries/clientpositive/limit_pushdown.q b/ql/src/test/queries/clientpositive/limit_pushdown.q
index ba52af4..4cbe501 100644
--- a/ql/src/test/queries/clientpositive/limit_pushdown.q
+++ b/ql/src/test/queries/clientpositive/limit_pushdown.q
@@ -14,8 +14,8 @@ select key,value from src order by key desc limit 20;
select key,value from src order by key desc limit 20;
explain
-select value, sum(key + 1) as sum from src group by value limit 20;
-select value, sum(key + 1) as sum from src group by value limit 20;
+select value, sum(key + 1) as sum from src group by value order by value limit 20;
+select value, sum(key + 1) as sum from src group by value order by value limit 20;
-- deduped RS
explain
@@ -28,13 +28,13 @@ select distinct(cdouble) from alltypesorc limit 20;
select distinct(cdouble) from alltypesorc limit 20;
explain
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20;
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20;
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20;
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20;
-- multi distinct
explain
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20;
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20;
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20;
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20;
-- limit zero
explain
@@ -57,8 +57,8 @@ on subq.key=subq2.key limit 4;
set hive.map.aggr=false;
-- map aggregation disabled
explain
-select value, sum(key) as sum from src group by value limit 20;
-select value, sum(key) as sum from src group by value limit 20;
+select value, sum(key) as sum from src group by value order by value limit 20;
+select value, sum(key) as sum from src group by value order by value limit 20;
set hive.limit.pushdown.memory.usage=0.00002f;
diff --git a/ql/src/test/queries/clientpositive/mapreduce2.q b/ql/src/test/queries/clientpositive/mapreduce2.q
index ef6f8e5..7539d3f 100644
--- a/ql/src/test/queries/clientpositive/mapreduce2.q
+++ b/ql/src/test/queries/clientpositive/mapreduce2.q
@@ -14,4 +14,4 @@ MAP src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value
USING 'cat' AS (tkey, ten, one, tvalue)
DISTRIBUTE BY tvalue, tkey;
-SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T;
+SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key;
diff --git a/ql/src/test/queries/clientpositive/multi_insert_gby3.q b/ql/src/test/queries/clientpositive/multi_insert_gby3.q
index 040adca..1221af4 100644
--- a/ql/src/test/queries/clientpositive/multi_insert_gby3.q
+++ b/ql/src/test/queries/clientpositive/multi_insert_gby3.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
create table e1 (key string, keyD double);
create table e2 (key string, keyD double, value string);
create table e3 (key string, keyD double);
diff --git a/ql/src/test/queries/clientpositive/multi_join_union.q b/ql/src/test/queries/clientpositive/multi_join_union.q
index 39650fd..c555b38 100644
--- a/ql/src/test/queries/clientpositive/multi_join_union.q
+++ b/ql/src/test/queries/clientpositive/multi_join_union.q
@@ -1,6 +1,7 @@
-
set hive.auto.convert.join=true;
+-- SORT_QUERY_RESULTS
+
CREATE TABLE src11 as SELECT * FROM src;
CREATE TABLE src12 as SELECT * FROM src;
CREATE TABLE src13 as SELECT * FROM src;
@@ -10,4 +11,4 @@ CREATE TABLE src14 as SELECT * FROM src;
EXPLAIN SELECT * FROM
src11 a JOIN
src12 b ON (a.key = b.key) JOIN
-(SELECT * FROM (SELECT * FROM src13 UNION ALL SELECT * FROM src14)a )c ON c.value = b.value;
\ No newline at end of file
+(SELECT * FROM (SELECT * FROM src13 UNION ALL SELECT * FROM src14)a )c ON c.value = b.value;
diff --git a/ql/src/test/queries/clientpositive/ppd_outer_join3.q b/ql/src/test/queries/clientpositive/ppd_outer_join3.q
index d756b90..03169a4 100644
--- a/ql/src/test/queries/clientpositive/ppd_outer_join3.q
+++ b/ql/src/test/queries/clientpositive/ppd_outer_join3.q
@@ -1,6 +1,8 @@
set hive.optimize.ppd=true;
set hive.ppd.remove.duplicatefilters=false;
+-- SORT_QUERY_RESULTS
+
EXPLAIN
FROM
src a
diff --git a/ql/src/test/queries/clientpositive/ptf_decimal.q b/ql/src/test/queries/clientpositive/ptf_decimal.q
index d48dd36..94885e3 100644
--- a/ql/src/test/queries/clientpositive/ptf_decimal.q
+++ b/ql/src/test/queries/clientpositive/ptf_decimal.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
-- 1. aggregate functions with decimal type
diff --git a/ql/src/test/queries/clientpositive/ptf_general_queries.q b/ql/src/test/queries/clientpositive/ptf_general_queries.q
index fcb2e41..9942063 100644
--- a/ql/src/test/queries/clientpositive/ptf_general_queries.q
+++ b/ql/src/test/queries/clientpositive/ptf_general_queries.q
@@ -1,3 +1,5 @@
+-- SORT_QUERY_RESULTS
+
-- 1. testNoPTFNoWindowing
select p_mfgr, p_name, p_size
from part
diff --git a/ql/src/test/results/clientpositive/spark/auto_join20.q.out b/ql/src/test/results/clientpositive/spark/auto_join20.q.out
index e089234..31ee1bc 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join20.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join20.q.out
@@ -249,7 +249,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_join29.q.out b/ql/src/test/results/clientpositive/spark/auto_join29.q.out
index 6702c1c..e618ce0 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join29.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join29.q.out
@@ -694,7 +694,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1327,7 +1327,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1969,7 +1969,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2606,7 +2606,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -2738,7 +2738,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3371,7 +3371,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -3504,7 +3504,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3654,7 +3654,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
diff --git a/ql/src/test/results/clientpositive/spark/auto_join30.q.out b/ql/src/test/results/clientpositive/spark/auto_join30.q.out
index f6dd445..b401af5 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join30.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join30.q.out
@@ -185,7 +185,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -221,7 +221,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -341,7 +341,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -376,7 +376,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -503,8 +503,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -579,7 +579,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -718,8 +718,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -788,7 +788,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -924,8 +924,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -994,7 +994,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1130,8 +1130,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1199,7 +1199,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1336,8 +1336,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1405,7 +1405,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_join32.q.out b/ql/src/test/results/clientpositive/spark/auto_join32.q.out
index 9a7a982..fff4d54 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join32.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join32.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -298,7 +298,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -444,7 +444,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
index f5e8f88..26d2165 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
@@ -1,8 +1,8 @@
PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key limit 40
+select a.* from src a join src b on a.key=b.key order by key, value limit 40
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key limit 40
+select a.* from src a join src b on a.key=b.key order by key, value limit 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -12,7 +12,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -28,7 +29,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: a
@@ -56,16 +57,26 @@ STAGE PLANS:
expressions: _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -73,11 +84,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select a.* from src a join src b on a.key=b.key limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -123,10 +134,10 @@ RUN: Stage-1:MAPRED
119 val_119
119 val_119
PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -136,8 +147,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -153,7 +165,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: c
@@ -166,7 +178,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: value (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 6
Map Operator Tree:
TableScan
alias: a
@@ -210,16 +222,26 @@ STAGE PLANS:
expressions: _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -227,11 +249,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -290,8 +312,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -390,14 +412,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
RUN: Stage-1:MAPRED
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
104 val_104
104 val_104
104 val_104
@@ -406,8 +420,6 @@ RUN: Stage-1:MAPRED
104 val_104
104 val_104
104 val_104
-105 val_105
-111 val_111
113 val_113
113 val_113
113 val_113
@@ -416,16 +428,26 @@ RUN: Stage-1:MAPRED
113 val_113
113 val_113
113 val_113
-114 val_114
116 val_116
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
119 val_119
119 val_119
119 val_119
@@ -441,14 +463,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
RUN: Stage-1:MAPRED
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
104 val_104
104 val_104
104 val_104
@@ -457,8 +471,6 @@ RUN: Stage-1:MAPRED
104 val_104
104 val_104
104 val_104
-105 val_105
-111 val_111
113 val_113
113 val_113
113 val_113
@@ -467,16 +479,26 @@ RUN: Stage-1:MAPRED
113 val_113
113 val_113
113 val_113
-114 val_114
116 val_116
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
119 val_119
119 val_119
119 val_119
diff --git a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
index 390af23..c66b49d 100644
--- a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
@@ -157,7 +157,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -300,9 +300,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1639,7 +1639,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
index 445958a..382c340 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
@@ -72,7 +72,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3)
Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
@@ -227,7 +227,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
index 4562430..731dd1a 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
@@ -96,7 +96,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -204,7 +204,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -312,7 +312,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -416,7 +416,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -613,7 +613,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -717,7 +717,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -821,7 +821,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1018,7 +1018,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
index d69ab4a..4853aee 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
@@ -151,7 +151,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -262,7 +262,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -405,9 +405,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1700,7 +1700,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1811,7 +1811,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1954,9 +1954,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
index 2790a18..b0a81e8 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
@@ -212,7 +212,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -348,7 +348,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -499,7 +499,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -852,7 +852,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -965,7 +965,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
index b6d05df..6d9cc20 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
@@ -592,7 +592,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -696,7 +696,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/count.q.out b/ql/src/test/results/clientpositive/spark/count.q.out
index 89d5544..3e4ac5c 100644
--- a/ql/src/test/results/clientpositive/spark/count.q.out
+++ b/ql/src/test/results/clientpositive/spark/count.q.out
@@ -1,8 +1,10 @@
-PREHOOK: query: create table abcd (a int, b int, c int, d int)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@abcd
-POSTHOOK: query: create table abcd (a int, b int, c int, d int)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@abcd
@@ -22,13 +24,13 @@ POSTHOOK: query: select * from abcd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 35 23 6
+10 100 45 4
+10 100 NULL 5
10 1000 50 1
100 100 10 3
-12 NULL 80 2
-10 100 NULL 5
-10 100 45 4
12 100 75 7
+12 NULL 80 2
+NULL 35 23 6
PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
PREHOOK: type: QUERY
POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
@@ -99,10 +101,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
@@ -184,7 +186,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -236,10 +238,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out b/ql/src/test/results/clientpositive/spark/ctas.q.out
index 2ee6d9f..15c46ed 100644
--- a/ql/src/test/results/clientpositive/spark/ctas.q.out
+++ b/ql/src/test/results/clientpositive/spark/ctas.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -320,7 +320,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -529,7 +529,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -716,7 +716,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/groupby10.q.out b/ql/src/test/results/clientpositive/spark/groupby10.q.out
index 47de08c..31ea336 100644
--- a/ql/src/test/results/clientpositive/spark/groupby10.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby10.q.out
@@ -268,10 +268,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -483,10 +483,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby8.q.out b/ql/src/test/results/clientpositive/spark/groupby8.q.out
index d943957..c0c0cf2 100644
--- a/ql/src/test/results/clientpositive/spark/groupby8.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby8.q.out
@@ -828,10 +828,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby9.q.out b/ql/src/test/results/clientpositive/spark/groupby9.q.out
index 308b777..2707239 100644
--- a/ql/src/test/results/clientpositive/spark/groupby9.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby9.q.out
@@ -829,10 +829,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1619,10 +1619,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2409,8 +2409,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (GROUP, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -3196,10 +3196,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
index 2b6585b..8ed3e67 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
@@ -122,7 +122,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -199,8 +199,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -304,7 +304,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -404,10 +404,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 6
diff --git a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
index c545d47..fc02ea3 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
@@ -225,7 +225,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -401,7 +401,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -577,7 +577,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
index 8e4e547..6367c43 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
@@ -238,8 +238,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -438,7 +438,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -569,7 +569,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 5 <- Map 4 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -618,7 +618,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
index 0563354..8905705 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
@@ -116,7 +116,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -193,8 +193,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -292,7 +292,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -392,10 +392,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 6
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
index 9507753..98bb89e 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
@@ -297,7 +297,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1184,7 +1184,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1411,7 +1411,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1674,7 +1674,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1992,7 +1992,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2094,7 +2094,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2179,7 +2179,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2323,7 +2323,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP, 3)
Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -2886,7 +2886,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2971,7 +2971,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3116,7 +3116,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3419,7 +3419,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3694,7 +3694,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
@@ -3777,7 +3777,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
index bb2f97f..59db1f1 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
@@ -297,8 +297,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1202,8 +1202,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1447,8 +1447,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1728,8 +1728,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2064,7 +2064,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2166,7 +2166,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2251,7 +2251,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2395,8 +2395,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP, 3)
Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -2976,7 +2976,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3061,7 +3061,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3206,8 +3206,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3527,8 +3527,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3820,7 +3820,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
@@ -3903,7 +3903,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
diff --git a/ql/src/test/results/clientpositive/spark/having.q.out b/ql/src/test/results/clientpositive/spark/having.q.out
index 5e9f20d..3b80bd1 100644
--- a/ql/src/test/results/clientpositive/spark/having.q.out
+++ b/ql/src/test/results/clientpositive/spark/having.q.out
@@ -95,7 +95,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -476,7 +476,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -748,7 +748,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -949,7 +949,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1221,7 +1221,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/join20.q.out b/ql/src/test/results/clientpositive/spark/join20.q.out
index a456fbc..d7c2c7c 100644
--- a/ql/src/test/results/clientpositive/spark/join20.q.out
+++ b/ql/src/test/results/clientpositive/spark/join20.q.out
@@ -751,7 +751,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join38.q.out b/ql/src/test/results/clientpositive/spark/join38.q.out
index d977b4d..a511b03 100644
--- a/ql/src/test/results/clientpositive/spark/join38.q.out
+++ b/ql/src/test/results/clientpositive/spark/join38.q.out
@@ -79,7 +79,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join40.q.out b/ql/src/test/results/clientpositive/spark/join40.q.out
index 86c54c1..a4a0dc6 100644
--- a/ql/src/test/results/clientpositive/spark/join40.q.out
+++ b/ql/src/test/results/clientpositive/spark/join40.q.out
@@ -1843,7 +1843,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2528,7 +2528,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out
index 07e3601..1a6f157 100644
--- a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out
@@ -1288,7 +1288,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
index ccc0808..d8443fb 100644
--- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
@@ -173,10 +173,10 @@ POSTHOOK: Input: default@src
98 val_98
98 val_98
PREHOOK: query: explain
-select value, sum(key + 1) as sum from src group by value limit 20
+select value, sum(key + 1) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select value, sum(key + 1) as sum from src group by value limit 20
+select value, sum(key + 1) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -186,7 +186,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -240,11 +240,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+PREHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+POSTHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -382,7 +382,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -462,10 +462,10 @@ POSTHOOK: Input: default@alltypesorc
-16379.0
NULL
PREHOOK: query: explain
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -528,11 +528,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -558,11 +558,11 @@ POSTHOOK: Input: default@alltypesorc
NULL 2932
PREHOOK: query: -- multi distinct
explain
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
POSTHOOK: query: -- multi distinct
explain
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -625,11 +625,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -695,7 +695,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -870,7 +870,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -966,11 +966,11 @@ STAGE PLANS:
PREHOOK: query: -- map aggregation disabled
explain
-select value, sum(key) as sum from src group by value limit 20
+select value, sum(key) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
POSTHOOK: query: -- map aggregation disabled
explain
-select value, sum(key) as sum from src group by value limit 20
+select value, sum(key) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -980,7 +980,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1028,20 +1028,23 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select value, sum(key) as sum from src group by value limit 20
+PREHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20
+POSTHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
+val_0 0.0
val_10 10.0
val_100 200.0
val_103 206.0
+val_104 208.0
val_105 105.0
val_11 11.0
val_111 111.0
+val_113 226.0
val_114 114.0
val_116 116.0
val_118 236.0
@@ -1049,13 +1052,10 @@ val_119 357.0
val_12 24.0
val_120 240.0
val_125 250.0
+val_126 126.0
val_128 384.0
+val_129 258.0
val_131 131.0
-val_137 137.0
-val_138 414.0
-val_15 15.0
-val_174 174.0
-val_193 193.0
PREHOOK: query: -- flush for order-by
explain
select key,value,value,value,value,value,value,value,value from src order by key limit 100
@@ -1237,7 +1237,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1293,103 +1293,103 @@ POSTHOOK: query: select sum(key) as sum from src group by concat(key,value,value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0.0
10.0
105.0
-11.0
-111.0
+1107.0
114.0
+1227.0
+1251.0
126.0
-131.0
+1314.0
133.0
136.0
143.0
145.0
-150.0
+1494.0
+15.0
153.0
155.0
-156.0
-157.0
-158.0
-160.0
-162.0
163.0
166.0
17.0
-170.0
-177.0
-178.0
-180.0
-181.0
183.0
-186.0
-189.0
-19.0
-192.0
+1872.0
+194.0
194.0
-196.0
-20.0
-200.0
+1956.0
201.0
202.0
-206.0
+208.0
+210.0
214.0
222.0
226.0
-226.0
-228.0
-24.0
-24.0
-240.0
-242.0
+2345.0
+241.0
250.0
-258.0
+252.0
+262.0
+263.0
268.0
-272.0
-273.0
-274.0
-280.0
-282.0
-298.0
-304.0
-309.0
-316.0
-327.0
+283.0
+291.0
+292.0
+30.0
+310.0
328.0
+33.0
330.0
-348.0
+339.0
+341.0
348.0
350.0
-352.0
-357.0
-358.0
-36.0
-382.0
-384.0
+351.0
+360.0
+379.0
+389.0
390.0
-394.0
-399.0
-400.0
+4.0
406.0
-417.0
+41.0
418.0
+419.0
426.0
-430.0
-432.0
+437.0
442.0
446.0
448.0
-458.0
-459.0
+448.0
+449.0
+457.0
466.0
-469.0
-474.0
-478.0
-489.0
-552.0
-561.0
+467.0
+477.0
+479.0
+484.0
+487.0
+496.0
+497.0
+53.0
+544.0
+562.0
+564.0
+57.0
579.0
-597.0
-622.0
-624.0
-636.0
+618.0
+642.0
+65.0
+662.0
+69.0
+74.0
+77.0
+798.0
+819.0
+82.0
+84.0
+85.0
+858.0
+878.0
+916.0
+918.0
+933.0
+956.0
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
index 07b0152..2096a8a 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
@@ -190,7 +190,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -326,8 +326,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -470,7 +470,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
index 287f19d..b5e58d5 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
@@ -112,7 +112,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -285,7 +285,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
index f458227..4bc10b5 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
@@ -289,7 +289,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1131,7 +1131,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out
index e85410b..7956fd0 100644
--- a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out
@@ -101,11 +101,11 @@ POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string,
POSTHOOK: Lineage: dest1.one SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.ten SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T
+PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T
+POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert.q.out b/ql/src/test/results/clientpositive/spark/multi_insert.q.out
index ebb3616..941c430 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert.q.out
@@ -587,7 +587,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -745,7 +745,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -903,7 +903,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1061,7 +1061,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
index 8d32943..5bbfe19 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
@@ -234,7 +234,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out
index a8f1818..fe29a5a 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out
@@ -1,8 +1,10 @@
-PREHOOK: query: create table e1 (key string, keyD double)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+create table e1 (key string, keyD double)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@e1
-POSTHOOK: query: create table e1 (key string, keyD double)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+create table e1 (key string, keyD double)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@e1
@@ -1605,8 +1607,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1748,7 +1750,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
index 08ad9a0..9c19cdd 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
@@ -291,8 +291,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (GROUP, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -563,8 +563,8 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (SORT, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -854,9 +854,9 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 5
@@ -1224,9 +1224,9 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 5
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
index 6a00b3b..ea17fa1 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
@@ -604,7 +604,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -766,7 +766,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -928,7 +928,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1090,7 +1090,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2417,10 +2417,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2556,10 +2556,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2695,10 +2695,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2834,10 +2834,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2981,10 +2981,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3231,10 +3231,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3481,10 +3481,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3731,10 +3731,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out
index 3bec983..e0fe067 100644
--- a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out
@@ -1,9 +1,13 @@
-PREHOOK: query: CREATE TABLE src11 as SELECT * FROM src
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE src11 as SELECT * FROM src
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@src11
-POSTHOOK: query: CREATE TABLE src11 as SELECT * FROM src
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE src11 as SELECT * FROM src
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
@@ -79,7 +83,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 1), Union 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 3), Union 3 (PARTITION-LEVEL SORT, 3)
Union 3 <- Map 2 (NONE, 0), Map 6 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
index 9a8dcf0..71439cd 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
@@ -75,7 +75,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
index eee3bc7..29acb9b 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
@@ -511,7 +511,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -894,7 +894,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1273,7 +1273,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
index 0978c4f..bbd671b 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
@@ -1,4 +1,6 @@
-PREHOOK: query: EXPLAIN
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN
FROM
src a
FULL OUTER JOIN
@@ -7,7 +9,9 @@ PREHOOK: query: EXPLAIN
SELECT a.key, a.value, b.key, b.value
WHERE a.key > '10' AND a.key < '20' AND b.key > '15' AND b.key < '25'
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN
FROM
src a
FULL OUTER JOIN
@@ -252,7 +256,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
index b8c32a8..6ee9f72 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
@@ -206,7 +206,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out
index fd51b53..8494e97 100644
--- a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out
+++ b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out
@@ -1,4 +1,6 @@
-PREHOOK: query: -- 1. aggregate functions with decimal type
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. aggregate functions with decimal type
select p_mfgr, p_retailprice,
lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1,
@@ -9,7 +11,9 @@ from part
PREHOOK: type: QUERY
PREHOOK: Input: default@part
#### A masked pattern was here ####
-POSTHOOK: query: -- 1. aggregate functions with decimal type
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. aggregate functions with decimal type
select p_mfgr, p_retailprice,
lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1,
@@ -22,30 +26,30 @@ POSTHOOK: Input: default@part
#### A masked pattern was here ####
Manufacturer#1 1173.15 1173.15 NULL 1173.15 1173.15
Manufacturer#1 1173.15 1753.76 1173.15 1173.15 1173.15
-Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76
-Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59
Manufacturer#1 1414.42 1632.66 1602.59 1173.15 1414.42
+Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59
Manufacturer#1 1632.66 NULL 1414.42 1173.15 1632.66
+Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76
Manufacturer#2 1690.68 1800.7 NULL 1690.68 1690.68
-Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7
-Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98
Manufacturer#2 1698.66 1701.6 2031.98 1690.68 1698.66
Manufacturer#2 1701.6 NULL 1698.66 1690.68 1701.6
-Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68
+Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7
+Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98
Manufacturer#3 1190.27 1410.39 1671.68 1671.68 1190.27
+Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29
Manufacturer#3 1410.39 1922.98 1190.27 1671.68 1410.39
+Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68
Manufacturer#3 1922.98 1337.29 1410.39 1671.68 1922.98
-Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29
-Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67
-Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42
Manufacturer#4 1206.26 1844.92 1375.42 1620.67 1206.26
-Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92
Manufacturer#4 1290.35 NULL 1844.92 1620.67 1290.35
-Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69
-Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66
-Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73
+Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42
+Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67
+Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92
Manufacturer#5 1018.1 1464.48 1788.73 1789.69 1018.1
Manufacturer#5 1464.48 NULL 1018.1 1789.69 1464.48
+Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66
+Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73
+Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69
PREHOOK: query: -- 2. ranking functions with decimal type
select p_mfgr, p_retailprice,
@@ -114,32 +118,32 @@ from part
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#1 1753.76 NULL
-Manufacturer#1 1632.66 1753.76
-Manufacturer#1 1602.59 1632.66
-Manufacturer#1 1414.42 1602.59
-Manufacturer#1 1173.15 1414.42
Manufacturer#1 1173.15 1173.15
-Manufacturer#2 2031.98 NULL
-Manufacturer#2 1800.7 2031.98
-Manufacturer#2 1701.6 1800.7
-Manufacturer#2 1698.66 1701.6
+Manufacturer#1 1173.15 1414.42
+Manufacturer#1 1414.42 1602.59
+Manufacturer#1 1602.59 1632.66
+Manufacturer#1 1632.66 1753.76
+Manufacturer#1 1753.76 NULL
Manufacturer#2 1690.68 1698.66
-Manufacturer#3 1922.98 NULL
-Manufacturer#3 1671.68 1922.98
-Manufacturer#3 1410.39 1671.68
-Manufacturer#3 1337.29 1410.39
+Manufacturer#2 1698.66 1701.6
+Manufacturer#2 1701.6 1800.7
+Manufacturer#2 1800.7 2031.98
+Manufacturer#2 2031.98 NULL
Manufacturer#3 1190.27 1337.29
-Manufacturer#4 1844.92 NULL
-Manufacturer#4 1620.67 1844.92
-Manufacturer#4 1375.42 1620.67
-Manufacturer#4 1290.35 1375.42
+Manufacturer#3 1337.29 1410.39
+Manufacturer#3 1410.39 1671.68
+Manufacturer#3 1671.68 1922.98
+Manufacturer#3 1922.98 NULL
Manufacturer#4 1206.26 1290.35
-Manufacturer#5 1789.69 NULL
-Manufacturer#5 1788.73 1789.69
-Manufacturer#5 1611.66 1788.73
-Manufacturer#5 1464.48 1611.66
+Manufacturer#4 1290.35 1375.42
+Manufacturer#4 1375.42 1620.67
+Manufacturer#4 1620.67 1844.92
+Manufacturer#4 1844.92 NULL
Manufacturer#5 1018.1 1464.48
+Manufacturer#5 1464.48 1611.66
+Manufacturer#5 1611.66 1788.73
+Manufacturer#5 1788.73 1789.69
+Manufacturer#5 1789.69 NULL
PREHOOK: query: -- 4. partition by decimal
select p_mfgr, p_retailprice,
@@ -156,29 +160,29 @@ from part
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#5 1018.1 NULL
-Manufacturer#1 1173.15 NULL
Manufacturer#1 1173.15 1173.15
-Manufacturer#3 1190.27 NULL
-Manufacturer#4 1206.26 NULL
-Manufacturer#4 1290.35 NULL
-Manufacturer#3 1337.29 NULL
-Manufacturer#4 1375.42 NULL
-Manufacturer#3 1410.39 NULL
+Manufacturer#1 1173.15 NULL
Manufacturer#1 1414.42 NULL
-Manufacturer#5 1464.48 NULL
Manufacturer#1 1602.59 NULL
-Manufacturer#5 1611.66 NULL
-Manufacturer#4 1620.67 NULL
Manufacturer#1 1632.66 NULL
-Manufacturer#3 1671.68 NULL
+Manufacturer#1 1753.76 NULL
Manufacturer#2 1690.68 NULL
Manufacturer#2 1698.66 NULL
Manufacturer#2 1701.6 NULL
-Manufacturer#1 1753.76 NULL
-Manufacturer#5 1788.73 NULL
-Manufacturer#5 1789.69 NULL
Manufacturer#2 1800.7 NULL
-Manufacturer#4 1844.92 NULL
-Manufacturer#3 1922.98 NULL
Manufacturer#2 2031.98 NULL
+Manufacturer#3 1190.27 NULL
+Manufacturer#3 1337.29 NULL
+Manufacturer#3 1410.39 NULL
+Manufacturer#3 1671.68 NULL
+Manufacturer#3 1922.98 NULL
+Manufacturer#4 1206.26 NULL
+Manufacturer#4 1290.35 NULL
+Manufacturer#4 1375.42 NULL
+Manufacturer#4 1620.67 NULL
+Manufacturer#4 1844.92 NULL
+Manufacturer#5 1018.1 NULL
+Manufacturer#5 1464.48 NULL
+Manufacturer#5 1611.66 NULL
+Manufacturer#5 1788.73 NULL
+Manufacturer#5 1789.69 NULL
diff --git a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out
index 25068da..a3d6194 100644
--- a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out
+++ b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out
@@ -1,4 +1,6 @@
-PREHOOK: query: -- 1. testNoPTFNoWindowing
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. testNoPTFNoWindowing
select p_mfgr, p_name, p_size
from part
distribute by p_mfgr
@@ -6,7 +8,9 @@ sort by p_name
PREHOOK: type: QUERY
PREHOOK: Input: default@part
#### A masked pattern was here ####
-POSTHOOK: query: -- 1. testNoPTFNoWindowing
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. testNoPTFNoWindowing
select p_mfgr, p_name, p_size
from part
distribute by p_mfgr
@@ -14,31 +18,31 @@ sort by p_name
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#5 almond antique blue firebrick mint 31
Manufacturer#1 almond antique burnished rose metallic 2
Manufacturer#1 almond antique burnished rose metallic 2
-Manufacturer#3 almond antique chartreuse khaki white 17
Manufacturer#1 almond antique chartreuse lavender yellow 34
-Manufacturer#3 almond antique forest lavender goldenrod 14
-Manufacturer#4 almond antique gainsboro frosted violet 10
-Manufacturer#5 almond antique medium spring khaki 6
-Manufacturer#3 almond antique metallic orange dim 19
-Manufacturer#3 almond antique misty red olive 1
-Manufacturer#3 almond antique olive coral navajo 45
Manufacturer#1 almond antique salmon chartreuse burlywood 6
-Manufacturer#5 almond antique sky peru orange 2
+Manufacturer#1 almond aquamarine burnished black steel 28
+Manufacturer#1 almond aquamarine pink moccasin thistle 42
Manufacturer#2 almond antique violet chocolate turquoise 14
-Manufacturer#4 almond antique violet mint lemon 39
Manufacturer#2 almond antique violet turquoise frosted 40
-Manufacturer#1 almond aquamarine burnished black steel 28
-Manufacturer#5 almond aquamarine dodger light gainsboro 46
-Manufacturer#4 almond aquamarine floral ivory bisque 27
Manufacturer#2 almond aquamarine midnight light salmon 2
-Manufacturer#1 almond aquamarine pink moccasin thistle 42
Manufacturer#2 almond aquamarine rose maroon antique 25
Manufacturer#2 almond aquamarine sandy cyan gainsboro 18
+Manufacturer#3 almond antique chartreuse khaki white 17
+Manufacturer#3 almond antique forest lavender goldenrod 14
+Manufacturer#3 almond antique metallic orange dim 19
+Manufacturer#3 almond antique misty red olive 1
+Manufacturer#3 almond antique olive coral navajo 45
+Manufacturer#4 almond antique gainsboro frosted violet 10
+Manufacturer#4 almond antique violet mint lemon 39
+Manufacturer#4 almond aquamarine floral ivory bisque 27
Manufacturer#4 almond aquamarine yellow dodger mint 7
Manufacturer#4 almond azure aquamarine papaya violet 12
+Manufacturer#5 almond antique blue firebrick mint 31
+Manufacturer#5 almond antique medium spring khaki 6
+Manufacturer#5 almond antique sky peru orange 2
+Manufacturer#5 almond aquamarine dodger light gainsboro 46
Manufacturer#5 almond azure blanched chiffon midnight 23
PREHOOK: query: -- 2. testUDAFsNoWindowingNoPTFNoGBY
select p_mfgr,p_name, p_retailprice,
diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out
index a9574ee..33cfe84 100644
--- a/ql/src/test/results/clientpositive/spark/sample10.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample10.q.out
@@ -89,7 +89,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out
index 9d527ce..d90d50d 100644
--- a/ql/src/test/results/clientpositive/spark/semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out
@@ -158,7 +158,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -272,7 +272,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -388,7 +388,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -496,7 +496,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -615,7 +615,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -726,7 +726,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -837,7 +837,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -945,7 +945,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1058,7 +1058,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1185,7 +1185,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1318,7 +1318,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1447,7 +1447,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1599,7 +1599,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1742,7 +1742,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1894,7 +1894,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2002,8 +2002,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2185,7 +2185,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -2340,7 +2340,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2450,8 +2450,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2645,7 +2645,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
index 4cecdf2..b0f0706 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
@@ -399,7 +399,7 @@ STAGE PLANS:
Spark
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -573,7 +573,7 @@ STAGE PLANS:
Spark
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
index 49405da..c35400a 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
@@ -214,8 +214,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
- Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+ Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
index ecb8a65..1019c2e 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
@@ -225,7 +225,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
index 4bbc455..fc4d7f0 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
@@ -208,7 +208,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -367,7 +367,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
index 49b9fda..6c736af 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
@@ -781,7 +781,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
index 363207a..b040a6a 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
@@ -623,7 +623,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
index 4bb6290..552b5ae 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
@@ -281,7 +281,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -441,8 +441,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -609,7 +609,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -829,7 +829,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out b/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out
index fbe392e..deaf5f2 100644
--- a/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out
+++ b/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out
@@ -209,7 +209,6 @@ Keys:key
2 1 2 12 1
3 1 3 13 1
7 1 7 17 1
-8 2 8 18 1
8 2 8 28 1
PREHOOK: query: -- constants from sub-queries should work fine
SELECT key, constant, val, count(1) from
diff --git a/ql/src/test/results/clientpositive/spark/union25.q.out b/ql/src/test/results/clientpositive/spark/union25.q.out
index abb93b6..b64ec53 100644
--- a/ql/src/test/results/clientpositive/spark/union25.q.out
+++ b/ql/src/test/results/clientpositive/spark/union25.q.out
@@ -68,8 +68,8 @@ STAGE PLANS:
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
Union 4 <- Map 7 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
- Reducer 5 <- Union 4 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
+ Reducer 5 <- Union 4 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out
index 0a81d4a..1fbc5b5 100644
--- a/ql/src/test/results/clientpositive/spark/union33.q.out
+++ b/ql/src/test/results/clientpositive/spark/union33.q.out
@@ -196,9 +196,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
index 41daaf6..d6d4add 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
@@ -276,8 +276,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -448,8 +448,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
index 5ff382f..e43a0ca 100644
--- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
index 7304a0c..279c61a 100644
--- a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
index 72cc00f..32bddd3 100644
--- a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
index b6a36d8..1927626 100644
--- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
@@ -257,7 +257,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
index 6a7c990..0992573 100644
--- a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
@@ -115,7 +115,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index d02c96e..269c4e3 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@ -1866,7 +1866,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2074,7 +2074,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2329,7 +2329,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2662,7 +2662,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
index 1028345..5fe0020 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
@@ -228,8 +228,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -588,8 +588,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -839,7 +839,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1095,8 +1095,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1383,8 +1383,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1680,9 +1680,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2020,7 +2020,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2263,7 +2263,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2589,8 +2589,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2866,8 +2866,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3147,8 +3147,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3435,9 +3435,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3762,8 +3762,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4222,8 +4222,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -4478,8 +4478,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4749,9 +4749,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -5222,11 +5222,11 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -5736,9 +5736,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6092,10 +6092,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6446,9 +6446,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6795,10 +6795,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -7194,9 +7194,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -7543,9 +7543,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/tez/count.q.out b/ql/src/test/results/clientpositive/tez/count.q.out
index fe63101..026b5ff 100644
--- a/ql/src/test/results/clientpositive/tez/count.q.out
+++ b/ql/src/test/results/clientpositive/tez/count.q.out
@@ -1,8 +1,10 @@
-PREHOOK: query: create table abcd (a int, b int, c int, d int)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@abcd
-POSTHOOK: query: create table abcd (a int, b int, c int, d int)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@abcd
@@ -22,13 +24,13 @@ POSTHOOK: query: select * from abcd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 35 23 6
+10 100 45 4
+10 100 NULL 5
10 1000 50 1
100 100 10 3
-12 NULL 80 2
-10 100 NULL 5
-10 100 45 4
12 100 75 7
+12 NULL 80 2
+NULL 35 23 6
PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
PREHOOK: type: QUERY
POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
@@ -99,10 +101,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
@@ -236,10 +238,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
diff --git a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
index c765ab7..d5d6e68 100644
--- a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
@@ -173,10 +173,10 @@ POSTHOOK: Input: default@src
98 val_98
98 val_98
PREHOOK: query: explain
-select value, sum(key + 1) as sum from src group by value limit 20
+select value, sum(key + 1) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select value, sum(key + 1) as sum from src group by value limit 20
+select value, sum(key + 1) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -240,11 +240,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+PREHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+POSTHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -462,10 +462,10 @@ POSTHOOK: Input: default@alltypesorc
-16379.0
NULL
PREHOOK: query: explain
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -528,11 +528,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -558,11 +558,11 @@ POSTHOOK: Input: default@alltypesorc
NULL 2932
PREHOOK: query: -- multi distinct
explain
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
POSTHOOK: query: -- multi distinct
explain
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -625,11 +625,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -958,11 +958,11 @@ STAGE PLANS:
PREHOOK: query: -- map aggregation disabled
explain
-select value, sum(key) as sum from src group by value limit 20
+select value, sum(key) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
POSTHOOK: query: -- map aggregation disabled
explain
-select value, sum(key) as sum from src group by value limit 20
+select value, sum(key) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -1020,11 +1020,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select value, sum(key) as sum from src group by value limit 20
+PREHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20
+POSTHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientpositive/tez/mapreduce2.q.out b/ql/src/test/results/clientpositive/tez/mapreduce2.q.out
index 373afe3..06c56d6 100644
--- a/ql/src/test/results/clientpositive/tez/mapreduce2.q.out
+++ b/ql/src/test/results/clientpositive/tez/mapreduce2.q.out
@@ -105,11 +105,11 @@ POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string,
POSTHOOK: Lineage: dest1.one SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.ten SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T
+PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T
+POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
index 47c13d1..98f5ad2 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
@@ -64,7 +64,7 @@ static ActorSystemInfo createActorSystem(Map conf) throws IOExce
int akkaHeartBeatPauses = toInt(conf.get("spark.akka.heartbeat.pauses"), 600);
double akkaFailureDetector =
toDouble(conf.get("spark.akka.failure-detector.threshold"), 300.0);
- int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 1000);
+ int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 100);
// Disabled due to chill-akka depending on kryo 2.21, which is incompatible with 2.22
// due to packaging changes (relocated org.objenesis classes).
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
index 0aea4a2..f2eac5e 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
@@ -17,10 +17,12 @@
package org.apache.hive.spark.client;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Serializable;
@@ -192,8 +194,12 @@ public void run() {
// If a Spark installation is provided, use the spark-submit script. Otherwise, call the
// SparkSubmit class directly, which has some caveats (like having to provide a proper
// version of Guava on the classpath depending on the deploy mode).
- if (conf.get("spark.home") != null) {
- argv.add(new File(conf.get("spark.home"), "bin/spark-submit").getAbsolutePath());
+ String sparkHome = conf.get("spark.home");
+ if (sparkHome == null) {
+ sparkHome = System.getProperty("spark.home");
+ }
+ if (sparkHome != null) {
+ argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath());
} else {
LOG.info("No spark.home provided, calling SparkSubmit directly.");
argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath());
@@ -252,8 +258,8 @@ public void run() {
final Process child = pb.start();
int childId = childIdGenerator.incrementAndGet();
- redirect("stdout-redir-" + childId, child.getInputStream(), System.out);
- redirect("stderr-redir-" + childId, child.getErrorStream(), System.err);
+ redirect("stdout-redir-" + childId, child.getInputStream());
+ redirect("stderr-redir-" + childId, child.getErrorStream());
runnable = new Runnable() {
@Override
@@ -277,8 +283,8 @@ public void run() {
return thread;
}
- private void redirect(String name, InputStream in, OutputStream out) {
- Thread thread = new Thread(new Redirector(in, out));
+ private void redirect(String name, InputStream in) {
+ Thread thread = new Thread(new Redirector(in));
thread.setName(name);
thread.setDaemon(true);
thread.start();
@@ -345,23 +351,18 @@ public void onReceive(Object message) throws Exception {
private class Redirector implements Runnable {
- private final InputStream in;
- private final OutputStream out;
+ private final BufferedReader in;
- Redirector(InputStream in, OutputStream out) {
- this.in = in;
- this.out = out;
+ Redirector(InputStream in) {
+ this.in = new BufferedReader(new InputStreamReader(in));
}
@Override
public void run() {
try {
- byte[] buf = new byte[1024];
- int len = in.read(buf);
- while (len != -1) {
- out.write(buf, 0, len);
- out.flush();
- len = in.read(buf);
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ LOG.info(line);
}
} catch (Exception e) {
LOG.warn("Error in redirector thread.", e);