diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties index f90b833..922efb8 100644 --- a/data/conf/hive-log4j.properties +++ b/data/conf/hive-log4j.properties @@ -42,7 +42,7 @@ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout # Pattern format: Date LogLevel LoggerName LogMessage #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n # Debugging Pattern format -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n # diff --git a/data/conf/spark/hive-site.xml b/data/conf/spark/hive-site.xml index f3d7b40..59c4cd9 100644 --- a/data/conf/spark/hive-site.xml +++ b/data/conf/spark/hive-site.xml @@ -197,7 +197,12 @@ spark.master - local[4] + local-cluster[2,2,1024] + + + + spark.akka.logLifecycleEvents + true diff --git a/itests/qtest-spark/.gitignore b/itests/qtest-spark/.gitignore new file mode 100644 index 0000000..c2ed135 --- /dev/null +++ b/itests/qtest-spark/.gitignore @@ -0,0 +1 @@ +thirdparty/ diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index a533e8d..fb121ff 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -339,6 +339,41 @@ maven-antrun-plugin + download-spark + generate-sources + + run + + + + + set -e + set -x + /bin/pwd + BASE_DIR=./target + DOWNLOAD_DIR=./thirdparty + download() { + url=$1; + finalName=$2 + tarName=$(basename $url) + rm -rf $BASE_DIR/$finalName + if [[ ! -f $DOWNLOAD_DIR/$tarName ]] + then + curl -Sso $DOWNLOAD_DIR/$tarName $url + fi + tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR + mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName + } + mkdir -p $DOWNLOAD_DIR + download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark" + + + + + + + + generate-tests-sources generate-test-sources diff --git a/pom.xml b/pom.xml index cce2e5e..900dddb 100644 --- a/pom.xml +++ b/pom.xml @@ -171,6 +171,7 @@ 2.4.0 2.6.0 3.0.0 + ${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark @@ -836,6 +837,7 @@ true ${test.tmp.dir} + ${spark.home} ${test.tmp.dir} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java index 72108cd..c075c35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java @@ -67,6 +67,9 @@ public int startMonitor() { while (true) { try { JobExecutionStatus state = sparkJobStatus.getState(); + if (LOG.isDebugEnabled()) { + console.printInfo("state = " + state); + } if (state != null && state != JobExecutionStatus.UNKNOWN && (state != lastState || state == JobExecutionStatus.RUNNING)) { lastState = state; diff --git a/ql/src/test/results/clientpositive/spark/auto_join20.q.out b/ql/src/test/results/clientpositive/spark/auto_join20.q.out index e089234..31ee1bc 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join20.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join20.q.out @@ -249,7 +249,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_join29.q.out b/ql/src/test/results/clientpositive/spark/auto_join29.q.out index 6702c1c..e618ce0 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join29.q.out @@ -694,7 +694,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1327,7 +1327,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1969,7 +1969,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2606,7 +2606,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -2738,7 +2738,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3371,7 +3371,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -3504,7 +3504,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3654,7 +3654,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 diff --git a/ql/src/test/results/clientpositive/spark/auto_join30.q.out b/ql/src/test/results/clientpositive/spark/auto_join30.q.out index f6dd445..b401af5 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join30.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join30.q.out @@ -185,7 +185,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -221,7 +221,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -341,7 +341,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -376,7 +376,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -503,8 +503,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -579,7 +579,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -718,8 +718,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -788,7 +788,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -924,8 +924,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -994,7 +994,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1130,8 +1130,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1199,7 +1199,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1336,8 +1336,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1405,7 +1405,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_join32.q.out b/ql/src/test/results/clientpositive/spark/auto_join32.q.out index 9a7a982..fff4d54 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join32.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join32.q.out @@ -176,7 +176,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -298,7 +298,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -444,7 +444,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out index 390af23..c66b49d 100644 --- a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out @@ -157,7 +157,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -300,9 +300,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1639,7 +1639,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out index 445958a..382c340 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out @@ -72,7 +72,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3) Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0) Reducer 4 <- Reducer 3 (GROUP, 1) #### A masked pattern was here #### @@ -227,7 +227,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out index 4562430..731dd1a 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out @@ -96,7 +96,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -204,7 +204,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -312,7 +312,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -416,7 +416,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -613,7 +613,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -717,7 +717,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -821,7 +821,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1018,7 +1018,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out index d69ab4a..4853aee 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out @@ -151,7 +151,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -262,7 +262,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -405,9 +405,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1700,7 +1700,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1811,7 +1811,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1954,9 +1954,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out index 2790a18..b0a81e8 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out @@ -212,7 +212,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -348,7 +348,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -499,7 +499,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -852,7 +852,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -965,7 +965,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out index b6d05df..6d9cc20 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out @@ -592,7 +592,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -696,7 +696,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out b/ql/src/test/results/clientpositive/spark/ctas.q.out index 2ee6d9f..15c46ed 100644 --- a/ql/src/test/results/clientpositive/spark/ctas.q.out +++ b/ql/src/test/results/clientpositive/spark/ctas.q.out @@ -176,7 +176,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -320,7 +320,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -529,7 +529,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -716,7 +716,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/groupby10.q.out b/ql/src/test/results/clientpositive/spark/groupby10.q.out index 47de08c..31ea336 100644 --- a/ql/src/test/results/clientpositive/spark/groupby10.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby10.q.out @@ -268,10 +268,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -483,10 +483,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby8.q.out b/ql/src/test/results/clientpositive/spark/groupby8.q.out index d943957..c0c0cf2 100644 --- a/ql/src/test/results/clientpositive/spark/groupby8.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby8.q.out @@ -828,10 +828,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby9.q.out b/ql/src/test/results/clientpositive/spark/groupby9.q.out index 308b777..2707239 100644 --- a/ql/src/test/results/clientpositive/spark/groupby9.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby9.q.out @@ -829,10 +829,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1619,10 +1619,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2409,8 +2409,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (GROUP, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -3196,10 +3196,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out index 2b6585b..8ed3e67 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out @@ -122,7 +122,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -199,8 +199,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -304,7 +304,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -404,10 +404,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 6 diff --git a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out index c545d47..fc02ea3 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out @@ -225,7 +225,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -401,7 +401,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -577,7 +577,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out index 8e4e547..6367c43 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out @@ -238,8 +238,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -438,7 +438,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -569,7 +569,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 5 <- Map 4 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -618,7 +618,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out index 0563354..8905705 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out @@ -116,7 +116,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -193,8 +193,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -292,7 +292,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -392,10 +392,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 6 diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 9507753..98bb89e 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -297,7 +297,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1184,7 +1184,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1411,7 +1411,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1674,7 +1674,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1992,7 +1992,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2094,7 +2094,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2179,7 +2179,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2323,7 +2323,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP, 3) Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -2886,7 +2886,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2971,7 +2971,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3116,7 +3116,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3419,7 +3419,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3694,7 +3694,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} @@ -3777,7 +3777,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index bb2f97f..59db1f1 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -297,8 +297,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1202,8 +1202,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1447,8 +1447,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1728,8 +1728,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2064,7 +2064,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2166,7 +2166,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2251,7 +2251,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2395,8 +2395,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP, 3) Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -2976,7 +2976,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3061,7 +3061,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3206,8 +3206,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3527,8 +3527,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3820,7 +3820,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} @@ -3903,7 +3903,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} diff --git a/ql/src/test/results/clientpositive/spark/having.q.out b/ql/src/test/results/clientpositive/spark/having.q.out index 5e9f20d..3b80bd1 100644 --- a/ql/src/test/results/clientpositive/spark/having.q.out +++ b/ql/src/test/results/clientpositive/spark/having.q.out @@ -95,7 +95,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -476,7 +476,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -748,7 +748,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -949,7 +949,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1221,7 +1221,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/join20.q.out b/ql/src/test/results/clientpositive/spark/join20.q.out index a456fbc..d7c2c7c 100644 --- a/ql/src/test/results/clientpositive/spark/join20.q.out +++ b/ql/src/test/results/clientpositive/spark/join20.q.out @@ -751,7 +751,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join38.q.out b/ql/src/test/results/clientpositive/spark/join38.q.out index d977b4d..a511b03 100644 --- a/ql/src/test/results/clientpositive/spark/join38.q.out +++ b/ql/src/test/results/clientpositive/spark/join38.q.out @@ -79,7 +79,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join40.q.out b/ql/src/test/results/clientpositive/spark/join40.q.out index 86c54c1..a4a0dc6 100644 --- a/ql/src/test/results/clientpositive/spark/join40.q.out +++ b/ql/src/test/results/clientpositive/spark/join40.q.out @@ -1843,7 +1843,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2528,7 +2528,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out index 07b0152..2096a8a 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out @@ -190,7 +190,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -326,8 +326,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -470,7 +470,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out index 287f19d..b5e58d5 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out @@ -112,7 +112,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -285,7 +285,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out index f458227..4bc10b5 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out @@ -289,7 +289,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1131,7 +1131,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert.q.out b/ql/src/test/results/clientpositive/spark/multi_insert.q.out index ebb3616..941c430 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert.q.out @@ -587,7 +587,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -745,7 +745,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -903,7 +903,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1061,7 +1061,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out index 8d32943..5bbfe19 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out @@ -234,7 +234,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out index 08ad9a0..9c19cdd 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out @@ -291,8 +291,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (GROUP, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -563,8 +563,8 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (SORT, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -854,9 +854,9 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: - Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 5 @@ -1224,9 +1224,9 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 5 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out index 6a00b3b..ea17fa1 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out @@ -604,7 +604,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -766,7 +766,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -928,7 +928,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1090,7 +1090,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2417,10 +2417,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2556,10 +2556,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2695,10 +2695,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2834,10 +2834,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2981,10 +2981,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3231,10 +3231,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3481,10 +3481,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3731,10 +3731,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out index 9a8dcf0..71439cd 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out @@ -75,7 +75,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out index eee3bc7..29acb9b 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out @@ -511,7 +511,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -894,7 +894,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1273,7 +1273,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out index b8c32a8..6ee9f72 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out @@ -206,7 +206,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out index a9574ee..33cfe84 100644 --- a/ql/src/test/results/clientpositive/spark/sample10.q.out +++ b/ql/src/test/results/clientpositive/spark/sample10.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out index 9d527ce..d90d50d 100644 --- a/ql/src/test/results/clientpositive/spark/semijoin.q.out +++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out @@ -158,7 +158,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -272,7 +272,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -388,7 +388,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -496,7 +496,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -615,7 +615,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -726,7 +726,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -837,7 +837,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -945,7 +945,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1058,7 +1058,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1185,7 +1185,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1318,7 +1318,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1447,7 +1447,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1599,7 +1599,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1742,7 +1742,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1894,7 +1894,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2002,8 +2002,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2185,7 +2185,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -2340,7 +2340,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2450,8 +2450,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2645,7 +2645,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out index 4cecdf2..b0f0706 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out @@ -399,7 +399,7 @@ STAGE PLANS: Spark Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -573,7 +573,7 @@ STAGE PLANS: Spark Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out index 49405da..c35400a 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out @@ -214,8 +214,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out index ecb8a65..1019c2e 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out @@ -225,7 +225,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out index 4bbc455..fc4d7f0 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out @@ -208,7 +208,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -367,7 +367,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) Reducer 4 <- Reducer 3 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out index 49b9fda..6c736af 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out @@ -781,7 +781,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out index 363207a..b040a6a 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out @@ -623,7 +623,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 4bb6290..552b5ae 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -281,7 +281,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -441,8 +441,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -609,7 +609,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -829,7 +829,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/union25.q.out b/ql/src/test/results/clientpositive/spark/union25.q.out index abb93b6..b64ec53 100644 --- a/ql/src/test/results/clientpositive/spark/union25.q.out +++ b/ql/src/test/results/clientpositive/spark/union25.q.out @@ -68,8 +68,8 @@ STAGE PLANS: Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) Union 4 <- Map 7 (NONE, 0), Reducer 3 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) - Reducer 5 <- Union 4 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) + Reducer 5 <- Union 4 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out index 0a81d4a..1fbc5b5 100644 --- a/ql/src/test/results/clientpositive/spark/union33.q.out +++ b/ql/src/test/results/clientpositive/spark/union33.q.out @@ -196,9 +196,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out index 41daaf6..d6d4add 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out @@ -276,8 +276,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -448,8 +448,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out index 5ff382f..e43a0ca 100644 --- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out index 7304a0c..279c61a 100644 --- a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out index 72cc00f..32bddd3 100644 --- a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out index b6a36d8..1927626 100644 --- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out @@ -257,7 +257,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out index 6a7c990..0992573 100644 --- a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out @@ -115,7 +115,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index d02c96e..269c4e3 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -1866,7 +1866,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2074,7 +2074,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2329,7 +2329,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2662,7 +2662,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 1028345..5fe0020 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -228,8 +228,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -588,8 +588,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -839,7 +839,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1095,8 +1095,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1383,8 +1383,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1680,9 +1680,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2020,7 +2020,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2263,7 +2263,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2589,8 +2589,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2866,8 +2866,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3147,8 +3147,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3435,9 +3435,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3762,8 +3762,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -4222,8 +4222,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -4478,8 +4478,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -4749,9 +4749,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -5222,11 +5222,11 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -5736,9 +5736,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6092,10 +6092,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6446,9 +6446,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6795,10 +6795,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -7194,9 +7194,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -7543,9 +7543,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java index 47c13d1..98f5ad2 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java @@ -64,7 +64,7 @@ static ActorSystemInfo createActorSystem(Map conf) throws IOExce int akkaHeartBeatPauses = toInt(conf.get("spark.akka.heartbeat.pauses"), 600); double akkaFailureDetector = toDouble(conf.get("spark.akka.failure-detector.threshold"), 300.0); - int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 1000); + int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 100); // Disabled due to chill-akka depending on kryo 2.21, which is incompatible with 2.22 // due to packaging changes (relocated org.objenesis classes). diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java index 0aea4a2..f2eac5e 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java @@ -17,10 +17,12 @@ package org.apache.hive.spark.client; +import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Serializable; @@ -192,8 +194,12 @@ public void run() { // If a Spark installation is provided, use the spark-submit script. Otherwise, call the // SparkSubmit class directly, which has some caveats (like having to provide a proper // version of Guava on the classpath depending on the deploy mode). - if (conf.get("spark.home") != null) { - argv.add(new File(conf.get("spark.home"), "bin/spark-submit").getAbsolutePath()); + String sparkHome = conf.get("spark.home"); + if (sparkHome == null) { + sparkHome = System.getProperty("spark.home"); + } + if (sparkHome != null) { + argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath()); } else { LOG.info("No spark.home provided, calling SparkSubmit directly."); argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath()); @@ -252,8 +258,8 @@ public void run() { final Process child = pb.start(); int childId = childIdGenerator.incrementAndGet(); - redirect("stdout-redir-" + childId, child.getInputStream(), System.out); - redirect("stderr-redir-" + childId, child.getErrorStream(), System.err); + redirect("stdout-redir-" + childId, child.getInputStream()); + redirect("stderr-redir-" + childId, child.getErrorStream()); runnable = new Runnable() { @Override @@ -277,8 +283,8 @@ public void run() { return thread; } - private void redirect(String name, InputStream in, OutputStream out) { - Thread thread = new Thread(new Redirector(in, out)); + private void redirect(String name, InputStream in) { + Thread thread = new Thread(new Redirector(in)); thread.setName(name); thread.setDaemon(true); thread.start(); @@ -345,23 +351,18 @@ public void onReceive(Object message) throws Exception { private class Redirector implements Runnable { - private final InputStream in; - private final OutputStream out; + private final BufferedReader in; - Redirector(InputStream in, OutputStream out) { - this.in = in; - this.out = out; + Redirector(InputStream in) { + this.in = new BufferedReader(new InputStreamReader(in)); } @Override public void run() { try { - byte[] buf = new byte[1024]; - int len = in.read(buf); - while (len != -1) { - out.write(buf, 0, len); - out.flush(); - len = in.read(buf); + String line = null; + while ((line = in.readLine()) != null) { + LOG.info(line); } } catch (Exception e) { LOG.warn("Error in redirector thread.", e);