diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties
index f90b833..922efb8 100644
--- a/data/conf/hive-log4j.properties
+++ b/data/conf/hive-log4j.properties
@@ -42,7 +42,7 @@ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
# Pattern format: Date LogLevel LoggerName LogMessage
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
#
diff --git a/data/conf/spark/hive-site.xml b/data/conf/spark/hive-site.xml
index f3d7b40..59c4cd9 100644
--- a/data/conf/spark/hive-site.xml
+++ b/data/conf/spark/hive-site.xml
@@ -197,7 +197,12 @@
spark.master
- local[4]
+ local-cluster[2,2,1024]
+
+
+
+ spark.akka.logLifecycleEvents
+ true
diff --git a/itests/qtest-spark/.gitignore b/itests/qtest-spark/.gitignore
new file mode 100644
index 0000000..c2ed135
--- /dev/null
+++ b/itests/qtest-spark/.gitignore
@@ -0,0 +1 @@
+thirdparty/
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index a533e8d..fb121ff 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -339,6 +339,41 @@
maven-antrun-plugin
+ download-spark
+ generate-sources
+
+ run
+
+
+
+
+ set -e
+ set -x
+ /bin/pwd
+ BASE_DIR=./target
+ DOWNLOAD_DIR=./thirdparty
+ download() {
+ url=$1;
+ finalName=$2
+ tarName=$(basename $url)
+ rm -rf $BASE_DIR/$finalName
+ if [[ ! -f $DOWNLOAD_DIR/$tarName ]]
+ then
+ curl -Sso $DOWNLOAD_DIR/$tarName $url
+ fi
+ tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
+ mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName
+ }
+ mkdir -p $DOWNLOAD_DIR
+ download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark"
+
+
+
+
+
+
+
+
generate-tests-sources
generate-test-sources
diff --git a/pom.xml b/pom.xml
index cce2e5e..900dddb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -171,6 +171,7 @@
2.4.0
2.6.0
3.0.0
+ ${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark
@@ -836,6 +837,7 @@
true
${test.tmp.dir}
+ ${spark.home}
${test.tmp.dir}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
index 72108cd..c075c35 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
@@ -67,6 +67,9 @@ public int startMonitor() {
while (true) {
try {
JobExecutionStatus state = sparkJobStatus.getState();
+ if (LOG.isDebugEnabled()) {
+ console.printInfo("state = " + state);
+ }
if (state != null && state != JobExecutionStatus.UNKNOWN &&
(state != lastState || state == JobExecutionStatus.RUNNING)) {
lastState = state;
diff --git a/ql/src/test/results/clientpositive/spark/auto_join20.q.out b/ql/src/test/results/clientpositive/spark/auto_join20.q.out
index e089234..31ee1bc 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join20.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join20.q.out
@@ -249,7 +249,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_join29.q.out b/ql/src/test/results/clientpositive/spark/auto_join29.q.out
index 6702c1c..e618ce0 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join29.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join29.q.out
@@ -694,7 +694,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1327,7 +1327,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1969,7 +1969,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2606,7 +2606,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -2738,7 +2738,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3371,7 +3371,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -3504,7 +3504,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3654,7 +3654,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
diff --git a/ql/src/test/results/clientpositive/spark/auto_join30.q.out b/ql/src/test/results/clientpositive/spark/auto_join30.q.out
index f6dd445..b401af5 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join30.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join30.q.out
@@ -185,7 +185,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -221,7 +221,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -341,7 +341,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -376,7 +376,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -503,8 +503,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -579,7 +579,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -718,8 +718,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -788,7 +788,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -924,8 +924,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -994,7 +994,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1130,8 +1130,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1199,7 +1199,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1336,8 +1336,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1405,7 +1405,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_join32.q.out b/ql/src/test/results/clientpositive/spark/auto_join32.q.out
index 9a7a982..fff4d54 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join32.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join32.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -298,7 +298,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -444,7 +444,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
index 390af23..c66b49d 100644
--- a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
@@ -157,7 +157,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -300,9 +300,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1639,7 +1639,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
index 445958a..382c340 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
@@ -72,7 +72,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3)
Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
@@ -227,7 +227,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
index 4562430..731dd1a 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
@@ -96,7 +96,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -204,7 +204,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -312,7 +312,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -416,7 +416,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -613,7 +613,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -717,7 +717,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -821,7 +821,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1018,7 +1018,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
index d69ab4a..4853aee 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
@@ -151,7 +151,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -262,7 +262,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -405,9 +405,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1700,7 +1700,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1811,7 +1811,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1954,9 +1954,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
index 2790a18..b0a81e8 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
@@ -212,7 +212,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -348,7 +348,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -499,7 +499,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -852,7 +852,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -965,7 +965,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
index b6d05df..6d9cc20 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
@@ -592,7 +592,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -696,7 +696,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out b/ql/src/test/results/clientpositive/spark/ctas.q.out
index 2ee6d9f..15c46ed 100644
--- a/ql/src/test/results/clientpositive/spark/ctas.q.out
+++ b/ql/src/test/results/clientpositive/spark/ctas.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -320,7 +320,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -529,7 +529,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -716,7 +716,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/groupby10.q.out b/ql/src/test/results/clientpositive/spark/groupby10.q.out
index 47de08c..31ea336 100644
--- a/ql/src/test/results/clientpositive/spark/groupby10.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby10.q.out
@@ -268,10 +268,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -483,10 +483,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby8.q.out b/ql/src/test/results/clientpositive/spark/groupby8.q.out
index d943957..c0c0cf2 100644
--- a/ql/src/test/results/clientpositive/spark/groupby8.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby8.q.out
@@ -828,10 +828,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby9.q.out b/ql/src/test/results/clientpositive/spark/groupby9.q.out
index 308b777..2707239 100644
--- a/ql/src/test/results/clientpositive/spark/groupby9.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby9.q.out
@@ -829,10 +829,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1619,10 +1619,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2409,8 +2409,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (GROUP, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -3196,10 +3196,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
index 2b6585b..8ed3e67 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
@@ -122,7 +122,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -199,8 +199,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -304,7 +304,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -404,10 +404,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 6
diff --git a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
index c545d47..fc02ea3 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
@@ -225,7 +225,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -401,7 +401,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -577,7 +577,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
index 8e4e547..6367c43 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
@@ -238,8 +238,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -438,7 +438,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -569,7 +569,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 5 <- Map 4 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -618,7 +618,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
index 0563354..8905705 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
@@ -116,7 +116,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -193,8 +193,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -292,7 +292,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -392,10 +392,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 6
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
index 9507753..98bb89e 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
@@ -297,7 +297,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1184,7 +1184,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1411,7 +1411,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1674,7 +1674,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1992,7 +1992,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2094,7 +2094,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2179,7 +2179,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2323,7 +2323,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP, 3)
Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -2886,7 +2886,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2971,7 +2971,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3116,7 +3116,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3419,7 +3419,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3694,7 +3694,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
@@ -3777,7 +3777,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
index bb2f97f..59db1f1 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
@@ -297,8 +297,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1202,8 +1202,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1447,8 +1447,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1728,8 +1728,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2064,7 +2064,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2166,7 +2166,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2251,7 +2251,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2395,8 +2395,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP, 3)
Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -2976,7 +2976,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3061,7 +3061,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3206,8 +3206,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3527,8 +3527,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3820,7 +3820,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
@@ -3903,7 +3903,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
diff --git a/ql/src/test/results/clientpositive/spark/having.q.out b/ql/src/test/results/clientpositive/spark/having.q.out
index 5e9f20d..3b80bd1 100644
--- a/ql/src/test/results/clientpositive/spark/having.q.out
+++ b/ql/src/test/results/clientpositive/spark/having.q.out
@@ -95,7 +95,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -476,7 +476,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -748,7 +748,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -949,7 +949,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1221,7 +1221,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/join20.q.out b/ql/src/test/results/clientpositive/spark/join20.q.out
index a456fbc..d7c2c7c 100644
--- a/ql/src/test/results/clientpositive/spark/join20.q.out
+++ b/ql/src/test/results/clientpositive/spark/join20.q.out
@@ -751,7 +751,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join38.q.out b/ql/src/test/results/clientpositive/spark/join38.q.out
index d977b4d..a511b03 100644
--- a/ql/src/test/results/clientpositive/spark/join38.q.out
+++ b/ql/src/test/results/clientpositive/spark/join38.q.out
@@ -79,7 +79,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join40.q.out b/ql/src/test/results/clientpositive/spark/join40.q.out
index 86c54c1..a4a0dc6 100644
--- a/ql/src/test/results/clientpositive/spark/join40.q.out
+++ b/ql/src/test/results/clientpositive/spark/join40.q.out
@@ -1843,7 +1843,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2528,7 +2528,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
index 07b0152..2096a8a 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
@@ -190,7 +190,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -326,8 +326,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -470,7 +470,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
index 287f19d..b5e58d5 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
@@ -112,7 +112,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -285,7 +285,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
index f458227..4bc10b5 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
@@ -289,7 +289,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1131,7 +1131,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert.q.out b/ql/src/test/results/clientpositive/spark/multi_insert.q.out
index ebb3616..941c430 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert.q.out
@@ -587,7 +587,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -745,7 +745,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -903,7 +903,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1061,7 +1061,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
index 8d32943..5bbfe19 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
@@ -234,7 +234,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
index 08ad9a0..9c19cdd 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
@@ -291,8 +291,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (GROUP, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -563,8 +563,8 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (SORT, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -854,9 +854,9 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 5
@@ -1224,9 +1224,9 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 5
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
index 6a00b3b..ea17fa1 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
@@ -604,7 +604,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -766,7 +766,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -928,7 +928,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1090,7 +1090,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2417,10 +2417,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2556,10 +2556,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2695,10 +2695,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2834,10 +2834,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2981,10 +2981,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3231,10 +3231,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3481,10 +3481,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3731,10 +3731,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
index 9a8dcf0..71439cd 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
@@ -75,7 +75,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
index eee3bc7..29acb9b 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
@@ -511,7 +511,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -894,7 +894,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1273,7 +1273,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
index b8c32a8..6ee9f72 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
@@ -206,7 +206,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out
index a9574ee..33cfe84 100644
--- a/ql/src/test/results/clientpositive/spark/sample10.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample10.q.out
@@ -89,7 +89,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out
index 9d527ce..d90d50d 100644
--- a/ql/src/test/results/clientpositive/spark/semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out
@@ -158,7 +158,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -272,7 +272,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -388,7 +388,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -496,7 +496,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -615,7 +615,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -726,7 +726,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -837,7 +837,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -945,7 +945,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1058,7 +1058,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1185,7 +1185,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1318,7 +1318,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1447,7 +1447,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1599,7 +1599,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1742,7 +1742,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1894,7 +1894,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2002,8 +2002,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2185,7 +2185,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -2340,7 +2340,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2450,8 +2450,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2645,7 +2645,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
index 4cecdf2..b0f0706 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
@@ -399,7 +399,7 @@ STAGE PLANS:
Spark
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -573,7 +573,7 @@ STAGE PLANS:
Spark
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
index 49405da..c35400a 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
@@ -214,8 +214,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
- Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+ Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
index ecb8a65..1019c2e 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
@@ -225,7 +225,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
index 4bbc455..fc4d7f0 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
@@ -208,7 +208,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -367,7 +367,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
index 49b9fda..6c736af 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
@@ -781,7 +781,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
index 363207a..b040a6a 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
@@ -623,7 +623,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
index 4bb6290..552b5ae 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
@@ -281,7 +281,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -441,8 +441,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -609,7 +609,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -829,7 +829,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/union25.q.out b/ql/src/test/results/clientpositive/spark/union25.q.out
index abb93b6..b64ec53 100644
--- a/ql/src/test/results/clientpositive/spark/union25.q.out
+++ b/ql/src/test/results/clientpositive/spark/union25.q.out
@@ -68,8 +68,8 @@ STAGE PLANS:
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
Union 4 <- Map 7 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
- Reducer 5 <- Union 4 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
+ Reducer 5 <- Union 4 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out
index 0a81d4a..1fbc5b5 100644
--- a/ql/src/test/results/clientpositive/spark/union33.q.out
+++ b/ql/src/test/results/clientpositive/spark/union33.q.out
@@ -196,9 +196,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
index 41daaf6..d6d4add 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
@@ -276,8 +276,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -448,8 +448,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
index 5ff382f..e43a0ca 100644
--- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
index 7304a0c..279c61a 100644
--- a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
index 72cc00f..32bddd3 100644
--- a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
index b6a36d8..1927626 100644
--- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
@@ -257,7 +257,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
index 6a7c990..0992573 100644
--- a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
@@ -115,7 +115,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index d02c96e..269c4e3 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@ -1866,7 +1866,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2074,7 +2074,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2329,7 +2329,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2662,7 +2662,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
index 1028345..5fe0020 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
@@ -228,8 +228,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -588,8 +588,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -839,7 +839,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1095,8 +1095,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1383,8 +1383,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1680,9 +1680,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2020,7 +2020,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2263,7 +2263,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2589,8 +2589,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2866,8 +2866,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3147,8 +3147,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3435,9 +3435,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3762,8 +3762,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4222,8 +4222,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -4478,8 +4478,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4749,9 +4749,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -5222,11 +5222,11 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -5736,9 +5736,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6092,10 +6092,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6446,9 +6446,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6795,10 +6795,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -7194,9 +7194,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -7543,9 +7543,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
index 47c13d1..98f5ad2 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
@@ -64,7 +64,7 @@ static ActorSystemInfo createActorSystem(Map conf) throws IOExce
int akkaHeartBeatPauses = toInt(conf.get("spark.akka.heartbeat.pauses"), 600);
double akkaFailureDetector =
toDouble(conf.get("spark.akka.failure-detector.threshold"), 300.0);
- int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 1000);
+ int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 100);
// Disabled due to chill-akka depending on kryo 2.21, which is incompatible with 2.22
// due to packaging changes (relocated org.objenesis classes).
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
index 0aea4a2..f2eac5e 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
@@ -17,10 +17,12 @@
package org.apache.hive.spark.client;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Serializable;
@@ -192,8 +194,12 @@ public void run() {
// If a Spark installation is provided, use the spark-submit script. Otherwise, call the
// SparkSubmit class directly, which has some caveats (like having to provide a proper
// version of Guava on the classpath depending on the deploy mode).
- if (conf.get("spark.home") != null) {
- argv.add(new File(conf.get("spark.home"), "bin/spark-submit").getAbsolutePath());
+ String sparkHome = conf.get("spark.home");
+ if (sparkHome == null) {
+ sparkHome = System.getProperty("spark.home");
+ }
+ if (sparkHome != null) {
+ argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath());
} else {
LOG.info("No spark.home provided, calling SparkSubmit directly.");
argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath());
@@ -252,8 +258,8 @@ public void run() {
final Process child = pb.start();
int childId = childIdGenerator.incrementAndGet();
- redirect("stdout-redir-" + childId, child.getInputStream(), System.out);
- redirect("stderr-redir-" + childId, child.getErrorStream(), System.err);
+ redirect("stdout-redir-" + childId, child.getInputStream());
+ redirect("stderr-redir-" + childId, child.getErrorStream());
runnable = new Runnable() {
@Override
@@ -277,8 +283,8 @@ public void run() {
return thread;
}
- private void redirect(String name, InputStream in, OutputStream out) {
- Thread thread = new Thread(new Redirector(in, out));
+ private void redirect(String name, InputStream in) {
+ Thread thread = new Thread(new Redirector(in));
thread.setName(name);
thread.setDaemon(true);
thread.start();
@@ -345,23 +351,18 @@ public void onReceive(Object message) throws Exception {
private class Redirector implements Runnable {
- private final InputStream in;
- private final OutputStream out;
+ private final BufferedReader in;
- Redirector(InputStream in, OutputStream out) {
- this.in = in;
- this.out = out;
+ Redirector(InputStream in) {
+ this.in = new BufferedReader(new InputStreamReader(in));
}
@Override
public void run() {
try {
- byte[] buf = new byte[1024];
- int len = in.read(buf);
- while (len != -1) {
- out.write(buf, 0, len);
- out.flush();
- len = in.read(buf);
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ LOG.info(line);
}
} catch (Exception e) {
LOG.warn("Error in redirector thread.", e);