diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties
index f90b833..922efb8 100644
--- a/data/conf/hive-log4j.properties
+++ b/data/conf/hive-log4j.properties
@@ -42,7 +42,7 @@ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
# Pattern format: Date LogLevel LoggerName LogMessage
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
#
diff --git a/data/conf/spark/hive-site.xml b/data/conf/spark/hive-site.xml
index f3d7b40..e42aa22 100644
--- a/data/conf/spark/hive-site.xml
+++ b/data/conf/spark/hive-site.xml
@@ -197,7 +197,17 @@
spark.master
- local[4]
+ local-cluster[2,2,1024]
+
+
+
+ spark.akka.logLifecycleEvents
+ true
+
+
+
+ spark.root.dir
+ ${spark.home}
diff --git a/data/conf/spark/log4j.properties b/data/conf/spark/log4j.properties
new file mode 100644
index 0000000..e884bdf
--- /dev/null
+++ b/data/conf/spark/log4j.properties
@@ -0,0 +1,30 @@
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, DRFA
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.eclipse.jetty=WARN
+log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
+
+
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+
+log4j.appender.DRFA.File=${spark.root.dir}/logs/spark.log
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
diff --git a/itests/qtest-spark/.gitignore b/itests/qtest-spark/.gitignore
new file mode 100644
index 0000000..c2ed135
--- /dev/null
+++ b/itests/qtest-spark/.gitignore
@@ -0,0 +1 @@
+thirdparty/
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index a533e8d..1b02be5 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -339,6 +339,44 @@
maven-antrun-plugin
+ download-spark
+ generate-sources
+
+ run
+
+
+
+
+ env
+ set -e
+ set -x
+ /bin/pwd
+ BASE_DIR=./target
+ HIVE_ROOT=$BASE_DIR/../../../
+ DOWNLOAD_DIR=./thirdparty
+ download() {
+ url=$1;
+ finalName=$2
+ tarName=$(basename $url)
+ rm -rf $BASE_DIR/$finalName
+ if [[ ! -f $DOWNLOAD_DIR/$tarName ]]
+ then
+ curl -Sso $DOWNLOAD_DIR/$tarName $url
+ fi
+ tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
+ mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName
+ }
+ mkdir -p $DOWNLOAD_DIR
+ download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark"
+ cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
+
+
+
+
+
+
+
+
generate-tests-sources
generate-test-sources
diff --git a/pom.xml b/pom.xml
index cce2e5e..900dddb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -171,6 +171,7 @@
2.4.0
2.6.0
3.0.0
+ ${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark
@@ -836,6 +837,7 @@
true
${test.tmp.dir}
+ ${spark.home}
${test.tmp.dir}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index 1898560..2ba75e0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -100,7 +100,7 @@ public static HiveSparkClient createHiveSparkClient(Configuration configuration)
for (Map.Entry entry : hiveConf) {
String propertyName = entry.getKey();
if (propertyName.startsWith("spark")) {
- String value = entry.getValue();
+ String value = hiveConf.get(propertyName);
sparkConf.put(propertyName, value);
LOG.info(String.format(
"load spark configuration from hive configuration (%s -> %s).",
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
index 72108cd..c075c35 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
@@ -67,6 +67,9 @@ public int startMonitor() {
while (true) {
try {
JobExecutionStatus state = sparkJobStatus.getState();
+ if (LOG.isDebugEnabled()) {
+ console.printInfo("state = " + state);
+ }
if (state != null && state != JobExecutionStatus.UNKNOWN &&
(state != lastState || state == JobExecutionStatus.RUNNING)) {
lastState = state;
diff --git a/ql/src/test/results/clientpositive/spark/auto_join20.q.out b/ql/src/test/results/clientpositive/spark/auto_join20.q.out
index e089234..31ee1bc 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join20.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join20.q.out
@@ -249,7 +249,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_join29.q.out b/ql/src/test/results/clientpositive/spark/auto_join29.q.out
index 6702c1c..e618ce0 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join29.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join29.q.out
@@ -694,7 +694,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1327,7 +1327,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1969,7 +1969,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2606,7 +2606,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -2738,7 +2738,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3371,7 +3371,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -3504,7 +3504,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3654,7 +3654,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
diff --git a/ql/src/test/results/clientpositive/spark/auto_join30.q.out b/ql/src/test/results/clientpositive/spark/auto_join30.q.out
index f6dd445..b401af5 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join30.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join30.q.out
@@ -185,7 +185,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -221,7 +221,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -341,7 +341,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -376,7 +376,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -503,8 +503,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -579,7 +579,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -718,8 +718,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -788,7 +788,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -924,8 +924,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -994,7 +994,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1130,8 +1130,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1199,7 +1199,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1336,8 +1336,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (SORT, 1)
- Reducer 7 <- Map 6 (SORT, 1)
+ Reducer 5 <- Map 4 (SORT, 3)
+ Reducer 7 <- Map 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1405,7 +1405,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_join32.q.out b/ql/src/test/results/clientpositive/spark/auto_join32.q.out
index 9a7a982..fff4d54 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join32.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join32.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -298,7 +298,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -444,7 +444,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
index f5e8f88..26d2165 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
@@ -1,8 +1,8 @@
PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key limit 40
+select a.* from src a join src b on a.key=b.key order by key, value limit 40
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key limit 40
+select a.* from src a join src b on a.key=b.key order by key, value limit 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -12,7 +12,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -28,7 +29,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: a
@@ -56,16 +57,26 @@ STAGE PLANS:
expressions: _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -73,11 +84,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select a.* from src a join src b on a.key=b.key limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -123,10 +134,10 @@ RUN: Stage-1:MAPRED
119 val_119
119 val_119
PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -136,8 +147,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -153,7 +165,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: c
@@ -166,7 +178,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: value (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 6
Map Operator Tree:
TableScan
alias: a
@@ -210,16 +222,26 @@ STAGE PLANS:
expressions: _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -227,11 +249,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -290,8 +312,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -390,14 +412,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
RUN: Stage-1:MAPRED
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
104 val_104
104 val_104
104 val_104
@@ -406,8 +420,6 @@ RUN: Stage-1:MAPRED
104 val_104
104 val_104
104 val_104
-105 val_105
-111 val_111
113 val_113
113 val_113
113 val_113
@@ -416,16 +428,26 @@ RUN: Stage-1:MAPRED
113 val_113
113 val_113
113 val_113
-114 val_114
116 val_116
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
119 val_119
119 val_119
119 val_119
@@ -441,14 +463,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
RUN: Stage-1:MAPRED
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
104 val_104
104 val_104
104 val_104
@@ -457,8 +471,6 @@ RUN: Stage-1:MAPRED
104 val_104
104 val_104
104 val_104
-105 val_105
-111 val_111
113 val_113
113 val_113
113 val_113
@@ -467,16 +479,26 @@ RUN: Stage-1:MAPRED
113 val_113
113 val_113
113 val_113
-114 val_114
116 val_116
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
-118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
119 val_119
119 val_119
119 val_119
diff --git a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
index 390af23..c66b49d 100644
--- a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out
@@ -157,7 +157,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -300,9 +300,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1639,7 +1639,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
index 445958a..382c340 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out
@@ -72,7 +72,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3)
Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
@@ -227,7 +227,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
index 4562430..731dd1a 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out
@@ -96,7 +96,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -204,7 +204,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -312,7 +312,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -416,7 +416,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -613,7 +613,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -717,7 +717,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -821,7 +821,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1018,7 +1018,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
index d69ab4a..4853aee 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out
@@ -151,7 +151,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -262,7 +262,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -405,9 +405,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1700,7 +1700,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1811,7 +1811,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1954,9 +1954,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
index 2790a18..b0a81e8 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
@@ -212,7 +212,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -348,7 +348,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -499,7 +499,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -852,7 +852,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -965,7 +965,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
index b6d05df..6d9cc20 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out
@@ -592,7 +592,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -696,7 +696,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/count.q.out b/ql/src/test/results/clientpositive/spark/count.q.out
index 89d5544..3e4ac5c 100644
--- a/ql/src/test/results/clientpositive/spark/count.q.out
+++ b/ql/src/test/results/clientpositive/spark/count.q.out
@@ -1,8 +1,10 @@
-PREHOOK: query: create table abcd (a int, b int, c int, d int)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@abcd
-POSTHOOK: query: create table abcd (a int, b int, c int, d int)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+create table abcd (a int, b int, c int, d int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@abcd
@@ -22,13 +24,13 @@ POSTHOOK: query: select * from abcd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 35 23 6
+10 100 45 4
+10 100 NULL 5
10 1000 50 1
100 100 10 3
-12 NULL 80 2
-10 100 NULL 5
-10 100 45 4
12 100 75 7
+12 NULL 80 2
+NULL 35 23 6
PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
PREHOOK: type: QUERY
POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
@@ -99,10 +101,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
@@ -184,7 +186,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -236,10 +238,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@abcd
#### A masked pattern was here ####
-NULL 1 1 6
10 2 2 10
-12 1 2 9
100 1 1 3
+12 1 2 9
+NULL 1 1 6
PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
PREHOOK: type: QUERY
POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out b/ql/src/test/results/clientpositive/spark/ctas.q.out
index 2ee6d9f..15c46ed 100644
--- a/ql/src/test/results/clientpositive/spark/ctas.q.out
+++ b/ql/src/test/results/clientpositive/spark/ctas.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -320,7 +320,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -529,7 +529,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -716,7 +716,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/groupby10.q.out b/ql/src/test/results/clientpositive/spark/groupby10.q.out
index 47de08c..31ea336 100644
--- a/ql/src/test/results/clientpositive/spark/groupby10.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby10.q.out
@@ -268,10 +268,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -483,10 +483,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby8.q.out b/ql/src/test/results/clientpositive/spark/groupby8.q.out
index d943957..c0c0cf2 100644
--- a/ql/src/test/results/clientpositive/spark/groupby8.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby8.q.out
@@ -828,10 +828,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby9.q.out b/ql/src/test/results/clientpositive/spark/groupby9.q.out
index 308b777..2707239 100644
--- a/ql/src/test/results/clientpositive/spark/groupby9.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby9.q.out
@@ -829,10 +829,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1619,10 +1619,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2409,8 +2409,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (GROUP, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -3196,10 +3196,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (GROUP, 1)
- Reducer 4 <- Reducer 6 (GROUP, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (GROUP, 3)
+ Reducer 4 <- Reducer 6 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
index 2b6585b..8ed3e67 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
@@ -122,7 +122,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -199,8 +199,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -304,7 +304,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -404,10 +404,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 6
diff --git a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
index c545d47..fc02ea3 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out
@@ -225,7 +225,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -401,7 +401,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -577,7 +577,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
index 8e4e547..6367c43 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
@@ -238,8 +238,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -438,7 +438,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -569,7 +569,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 5 <- Map 4 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -618,7 +618,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
index 0563354..8905705 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
@@ -116,7 +116,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -193,8 +193,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -292,7 +292,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -392,10 +392,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 6
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
index 9507753..98bb89e 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
@@ -297,7 +297,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1184,7 +1184,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1411,7 +1411,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1674,7 +1674,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1992,7 +1992,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2094,7 +2094,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2179,7 +2179,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2323,7 +2323,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP, 3)
Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -2886,7 +2886,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2971,7 +2971,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3116,7 +3116,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3419,7 +3419,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3694,7 +3694,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
@@ -3777,7 +3777,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
index bb2f97f..59db1f1 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
@@ -297,8 +297,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1202,8 +1202,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1447,8 +1447,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1728,8 +1728,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2064,7 +2064,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2166,7 +2166,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2251,7 +2251,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 1
+ numFiles 3
numRows 5
rawDataSize 17
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -2395,8 +2395,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (GROUP, 3)
Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -2976,7 +2976,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3061,7 +3061,7 @@ STAGE PLANS:
columns.types int:int
#### A masked pattern was here ####
name default.outputtbl1
- numFiles 2
+ numFiles 4
numRows 10
rawDataSize 32
serialization.ddl struct outputtbl1 { i32 key, i32 cnt}
@@ -3206,8 +3206,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -3527,8 +3527,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3820,7 +3820,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
@@ -3903,7 +3903,7 @@ STAGE PLANS:
columns.types int:int:string:int
#### A masked pattern was here ####
name default.outputtbl4
- numFiles 1
+ numFiles 3
numRows 6
rawDataSize 48
serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt}
diff --git a/ql/src/test/results/clientpositive/spark/having.q.out b/ql/src/test/results/clientpositive/spark/having.q.out
index 5e9f20d..3b80bd1 100644
--- a/ql/src/test/results/clientpositive/spark/having.q.out
+++ b/ql/src/test/results/clientpositive/spark/having.q.out
@@ -95,7 +95,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -476,7 +476,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -748,7 +748,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -949,7 +949,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1221,7 +1221,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/join20.q.out b/ql/src/test/results/clientpositive/spark/join20.q.out
index a456fbc..d7c2c7c 100644
--- a/ql/src/test/results/clientpositive/spark/join20.q.out
+++ b/ql/src/test/results/clientpositive/spark/join20.q.out
@@ -751,7 +751,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join38.q.out b/ql/src/test/results/clientpositive/spark/join38.q.out
index d977b4d..a511b03 100644
--- a/ql/src/test/results/clientpositive/spark/join38.q.out
+++ b/ql/src/test/results/clientpositive/spark/join38.q.out
@@ -79,7 +79,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join40.q.out b/ql/src/test/results/clientpositive/spark/join40.q.out
index 86c54c1..a4a0dc6 100644
--- a/ql/src/test/results/clientpositive/spark/join40.q.out
+++ b/ql/src/test/results/clientpositive/spark/join40.q.out
@@ -1843,7 +1843,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2528,7 +2528,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out
index 07e3601..1a6f157 100644
--- a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out
@@ -1288,7 +1288,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/join_reorder.q.out b/ql/src/test/results/clientpositive/spark/join_reorder.q.out
index abe0996..54522f9 100644
--- a/ql/src/test/results/clientpositive/spark/join_reorder.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_reorder.q.out
@@ -46,10 +46,14 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@t3
-PREHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key
SELECT a.key, a.val, c.key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key
SELECT a.key, a.val, c.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -475,9 +479,9 @@ POSTHOOK: Input: default@t2
POSTHOOK: Input: default@t3
#### A masked pattern was here ####
2 2 12 12
+7 NULL 17 17
NULL NULL NULL 14
NULL NULL NULL 16
-7 NULL 17 17
PREHOOK: query: FROM T1 a
LEFT OUTER JOIN T2 b ON (b.key=a.key)
RIGHT OUTER JOIN T3 c ON (c.val = a.val)
@@ -497,9 +501,9 @@ POSTHOOK: Input: default@t2
POSTHOOK: Input: default@t3
#### A masked pattern was here ####
2 2 12 12
+7 NULL 17 17
NULL NULL NULL 14
NULL NULL NULL 16
-7 NULL 17 17
PREHOOK: query: EXPLAIN FROM UNIQUEJOIN
PRESERVE T1 a (a.key, a.val),
PRESERVE T2 b (b.key, b.val),
@@ -520,7 +524,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -604,7 +608,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -690,15 +694,15 @@ POSTHOOK: Input: default@t3
#### A masked pattern was here ####
1 NULL NULL
2 NULL 2
-NULL 2 NULL
3 3 NULL
-NULL 4 4
-NULL 5 NULL
-NULL NULL 6
7 NULL 7
8 8 NULL
8 8 NULL
8 NULL NULL
+NULL 2 NULL
+NULL 4 4
+NULL 5 NULL
+NULL NULL 6
PREHOOK: query: FROM UNIQUEJOIN
PRESERVE T1 a (a.key, a.val),
PRESERVE T2 b (b.key, b.val),
@@ -721,12 +725,12 @@ POSTHOOK: Input: default@t3
#### A masked pattern was here ####
1 NULL NULL
2 NULL 2
-NULL 2 NULL
3 3 NULL
-NULL 4 4
-NULL 5 NULL
-NULL NULL 6
7 NULL 7
8 8 NULL
8 8 NULL
8 NULL NULL
+NULL 2 NULL
+NULL 4 4
+NULL 5 NULL
+NULL NULL 6
diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
index ccc0808..d8443fb 100644
--- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
@@ -173,10 +173,10 @@ POSTHOOK: Input: default@src
98 val_98
98 val_98
PREHOOK: query: explain
-select value, sum(key + 1) as sum from src group by value limit 20
+select value, sum(key + 1) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select value, sum(key + 1) as sum from src group by value limit 20
+select value, sum(key + 1) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -186,7 +186,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -240,11 +240,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+PREHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+POSTHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -382,7 +382,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -462,10 +462,10 @@ POSTHOOK: Input: default@alltypesorc
-16379.0
NULL
PREHOOK: query: explain
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -528,11 +528,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20
+POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -558,11 +558,11 @@ POSTHOOK: Input: default@alltypesorc
NULL 2932
PREHOOK: query: -- multi distinct
explain
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
POSTHOOK: query: -- multi distinct
explain
-select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -625,11 +625,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20
+POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -695,7 +695,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -870,7 +870,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 1)
+ Reducer 4 <- Map 3 (GROUP, 3)
Reducer 5 <- Reducer 4 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -966,11 +966,11 @@ STAGE PLANS:
PREHOOK: query: -- map aggregation disabled
explain
-select value, sum(key) as sum from src group by value limit 20
+select value, sum(key) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
POSTHOOK: query: -- map aggregation disabled
explain
-select value, sum(key) as sum from src group by value limit 20
+select value, sum(key) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -980,7 +980,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1028,20 +1028,23 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select value, sum(key) as sum from src group by value limit 20
+PREHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20
+POSTHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
+val_0 0.0
val_10 10.0
val_100 200.0
val_103 206.0
+val_104 208.0
val_105 105.0
val_11 11.0
val_111 111.0
+val_113 226.0
val_114 114.0
val_116 116.0
val_118 236.0
@@ -1049,13 +1052,10 @@ val_119 357.0
val_12 24.0
val_120 240.0
val_125 250.0
+val_126 126.0
val_128 384.0
+val_129 258.0
val_131 131.0
-val_137 137.0
-val_138 414.0
-val_15 15.0
-val_174 174.0
-val_193 193.0
PREHOOK: query: -- flush for order-by
explain
select key,value,value,value,value,value,value,value,value from src order by key limit 100
@@ -1237,7 +1237,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1293,103 +1293,103 @@ POSTHOOK: query: select sum(key) as sum from src group by concat(key,value,value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0.0
10.0
105.0
-11.0
-111.0
+1107.0
114.0
+1227.0
+1251.0
126.0
-131.0
+1314.0
133.0
136.0
143.0
145.0
-150.0
+1494.0
+15.0
153.0
155.0
-156.0
-157.0
-158.0
-160.0
-162.0
163.0
166.0
17.0
-170.0
-177.0
-178.0
-180.0
-181.0
183.0
-186.0
-189.0
-19.0
-192.0
+1872.0
+194.0
194.0
-196.0
-20.0
-200.0
+1956.0
201.0
202.0
-206.0
+208.0
+210.0
214.0
222.0
226.0
-226.0
-228.0
-24.0
-24.0
-240.0
-242.0
+2345.0
+241.0
250.0
-258.0
+252.0
+262.0
+263.0
268.0
-272.0
-273.0
-274.0
-280.0
-282.0
-298.0
-304.0
-309.0
-316.0
-327.0
+283.0
+291.0
+292.0
+30.0
+310.0
328.0
+33.0
330.0
-348.0
+339.0
+341.0
348.0
350.0
-352.0
-357.0
-358.0
-36.0
-382.0
-384.0
+351.0
+360.0
+379.0
+389.0
390.0
-394.0
-399.0
-400.0
+4.0
406.0
-417.0
+41.0
418.0
+419.0
426.0
-430.0
-432.0
+437.0
442.0
446.0
448.0
-458.0
-459.0
+448.0
+449.0
+457.0
466.0
-469.0
-474.0
-478.0
-489.0
-552.0
-561.0
+467.0
+477.0
+479.0
+484.0
+487.0
+496.0
+497.0
+53.0
+544.0
+562.0
+564.0
+57.0
579.0
-597.0
-622.0
-624.0
-636.0
+618.0
+642.0
+65.0
+662.0
+69.0
+74.0
+77.0
+798.0
+819.0
+82.0
+84.0
+85.0
+858.0
+878.0
+916.0
+918.0
+933.0
+956.0
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
index 07b0152..2096a8a 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
@@ -190,7 +190,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -326,8 +326,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -470,7 +470,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
index 287f19d..b5e58d5 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
@@ -112,7 +112,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -285,7 +285,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
index f458227..4bc10b5 100644
--- a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
@@ -289,7 +289,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1131,7 +1131,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out
index e85410b..7956fd0 100644
--- a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out
+++ b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out
@@ -101,11 +101,11 @@ POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string,
POSTHOOK: Lineage: dest1.one SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.ten SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T
+PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T
+POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert.q.out b/ql/src/test/results/clientpositive/spark/multi_insert.q.out
index ebb3616..941c430 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert.q.out
@@ -587,7 +587,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -745,7 +745,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -903,7 +903,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1061,7 +1061,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
index 8d32943..5bbfe19 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out
@@ -234,7 +234,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out
index a8f1818..fe29a5a 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out
@@ -1,8 +1,10 @@
-PREHOOK: query: create table e1 (key string, keyD double)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+create table e1 (key string, keyD double)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@e1
-POSTHOOK: query: create table e1 (key string, keyD double)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+create table e1 (key string, keyD double)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@e1
@@ -1605,8 +1607,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -1748,7 +1750,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
index 08ad9a0..9c19cdd 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
@@ -291,8 +291,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (GROUP, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -563,8 +563,8 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP, 1)
- Reducer 3 <- Map 5 (SORT, 1)
+ Reducer 2 <- Map 4 (GROUP, 3)
+ Reducer 3 <- Map 5 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 4
@@ -854,9 +854,9 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 5
@@ -1224,9 +1224,9 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 5
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
index 6a00b3b..ea17fa1 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
@@ -604,7 +604,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -766,7 +766,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -928,7 +928,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1090,7 +1090,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2417,10 +2417,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2556,10 +2556,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2695,10 +2695,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2834,10 +2834,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2981,10 +2981,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3231,10 +3231,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3481,10 +3481,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3731,10 +3731,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 5 <- Map 1 (SORT, 1)
- Reducer 6 <- Map 1 (SORT, 1)
- Reducer 3 <- Reducer 5 (SORT, 1)
- Reducer 4 <- Reducer 6 (SORT, 1)
+ Reducer 5 <- Map 1 (SORT, 3)
+ Reducer 6 <- Map 1 (SORT, 3)
+ Reducer 3 <- Reducer 5 (SORT, 3)
+ Reducer 4 <- Reducer 6 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out
index 3bec983..e0fe067 100644
--- a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out
@@ -1,9 +1,13 @@
-PREHOOK: query: CREATE TABLE src11 as SELECT * FROM src
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE src11 as SELECT * FROM src
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@src11
-POSTHOOK: query: CREATE TABLE src11 as SELECT * FROM src
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE src11 as SELECT * FROM src
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
@@ -79,7 +83,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 1), Union 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 3), Union 3 (PARTITION-LEVEL SORT, 3)
Union 3 <- Map 2 (NONE, 0), Map 6 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
index 97b6c6c..ad358ad 100644
--- a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
+++ b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
@@ -1,4 +1,6 @@
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN EXTENDED
FROM
src a
FULL OUTER JOIN
@@ -7,7 +9,9 @@ PREHOOK: query: EXPLAIN EXTENDED
SELECT a.key, a.value, b.key, b.value
WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN EXTENDED
FROM
src a
FULL OUTER JOIN
@@ -584,7 +588,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
index 9a8dcf0..71439cd 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out
@@ -75,7 +75,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
index eee3bc7..29acb9b 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
@@ -511,7 +511,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -894,7 +894,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1273,7 +1273,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
index 0978c4f..bbd671b 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
@@ -1,4 +1,6 @@
-PREHOOK: query: EXPLAIN
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN
FROM
src a
FULL OUTER JOIN
@@ -7,7 +9,9 @@ PREHOOK: query: EXPLAIN
SELECT a.key, a.value, b.key, b.value
WHERE a.key > '10' AND a.key < '20' AND b.key > '15' AND b.key < '25'
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN
FROM
src a
FULL OUTER JOIN
@@ -252,7 +256,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
index b8c32a8..6ee9f72 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out
@@ -206,7 +206,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out
index fd51b53..8494e97 100644
--- a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out
+++ b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out
@@ -1,4 +1,6 @@
-PREHOOK: query: -- 1. aggregate functions with decimal type
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. aggregate functions with decimal type
select p_mfgr, p_retailprice,
lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1,
@@ -9,7 +11,9 @@ from part
PREHOOK: type: QUERY
PREHOOK: Input: default@part
#### A masked pattern was here ####
-POSTHOOK: query: -- 1. aggregate functions with decimal type
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. aggregate functions with decimal type
select p_mfgr, p_retailprice,
lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1,
@@ -22,30 +26,30 @@ POSTHOOK: Input: default@part
#### A masked pattern was here ####
Manufacturer#1 1173.15 1173.15 NULL 1173.15 1173.15
Manufacturer#1 1173.15 1753.76 1173.15 1173.15 1173.15
-Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76
-Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59
Manufacturer#1 1414.42 1632.66 1602.59 1173.15 1414.42
+Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59
Manufacturer#1 1632.66 NULL 1414.42 1173.15 1632.66
+Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76
Manufacturer#2 1690.68 1800.7 NULL 1690.68 1690.68
-Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7
-Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98
Manufacturer#2 1698.66 1701.6 2031.98 1690.68 1698.66
Manufacturer#2 1701.6 NULL 1698.66 1690.68 1701.6
-Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68
+Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7
+Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98
Manufacturer#3 1190.27 1410.39 1671.68 1671.68 1190.27
+Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29
Manufacturer#3 1410.39 1922.98 1190.27 1671.68 1410.39
+Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68
Manufacturer#3 1922.98 1337.29 1410.39 1671.68 1922.98
-Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29
-Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67
-Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42
Manufacturer#4 1206.26 1844.92 1375.42 1620.67 1206.26
-Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92
Manufacturer#4 1290.35 NULL 1844.92 1620.67 1290.35
-Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69
-Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66
-Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73
+Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42
+Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67
+Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92
Manufacturer#5 1018.1 1464.48 1788.73 1789.69 1018.1
Manufacturer#5 1464.48 NULL 1018.1 1789.69 1464.48
+Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66
+Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73
+Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69
PREHOOK: query: -- 2. ranking functions with decimal type
select p_mfgr, p_retailprice,
@@ -114,32 +118,32 @@ from part
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#1 1753.76 NULL
-Manufacturer#1 1632.66 1753.76
-Manufacturer#1 1602.59 1632.66
-Manufacturer#1 1414.42 1602.59
-Manufacturer#1 1173.15 1414.42
Manufacturer#1 1173.15 1173.15
-Manufacturer#2 2031.98 NULL
-Manufacturer#2 1800.7 2031.98
-Manufacturer#2 1701.6 1800.7
-Manufacturer#2 1698.66 1701.6
+Manufacturer#1 1173.15 1414.42
+Manufacturer#1 1414.42 1602.59
+Manufacturer#1 1602.59 1632.66
+Manufacturer#1 1632.66 1753.76
+Manufacturer#1 1753.76 NULL
Manufacturer#2 1690.68 1698.66
-Manufacturer#3 1922.98 NULL
-Manufacturer#3 1671.68 1922.98
-Manufacturer#3 1410.39 1671.68
-Manufacturer#3 1337.29 1410.39
+Manufacturer#2 1698.66 1701.6
+Manufacturer#2 1701.6 1800.7
+Manufacturer#2 1800.7 2031.98
+Manufacturer#2 2031.98 NULL
Manufacturer#3 1190.27 1337.29
-Manufacturer#4 1844.92 NULL
-Manufacturer#4 1620.67 1844.92
-Manufacturer#4 1375.42 1620.67
-Manufacturer#4 1290.35 1375.42
+Manufacturer#3 1337.29 1410.39
+Manufacturer#3 1410.39 1671.68
+Manufacturer#3 1671.68 1922.98
+Manufacturer#3 1922.98 NULL
Manufacturer#4 1206.26 1290.35
-Manufacturer#5 1789.69 NULL
-Manufacturer#5 1788.73 1789.69
-Manufacturer#5 1611.66 1788.73
-Manufacturer#5 1464.48 1611.66
+Manufacturer#4 1290.35 1375.42
+Manufacturer#4 1375.42 1620.67
+Manufacturer#4 1620.67 1844.92
+Manufacturer#4 1844.92 NULL
Manufacturer#5 1018.1 1464.48
+Manufacturer#5 1464.48 1611.66
+Manufacturer#5 1611.66 1788.73
+Manufacturer#5 1788.73 1789.69
+Manufacturer#5 1789.69 NULL
PREHOOK: query: -- 4. partition by decimal
select p_mfgr, p_retailprice,
@@ -156,29 +160,29 @@ from part
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#5 1018.1 NULL
-Manufacturer#1 1173.15 NULL
Manufacturer#1 1173.15 1173.15
-Manufacturer#3 1190.27 NULL
-Manufacturer#4 1206.26 NULL
-Manufacturer#4 1290.35 NULL
-Manufacturer#3 1337.29 NULL
-Manufacturer#4 1375.42 NULL
-Manufacturer#3 1410.39 NULL
+Manufacturer#1 1173.15 NULL
Manufacturer#1 1414.42 NULL
-Manufacturer#5 1464.48 NULL
Manufacturer#1 1602.59 NULL
-Manufacturer#5 1611.66 NULL
-Manufacturer#4 1620.67 NULL
Manufacturer#1 1632.66 NULL
-Manufacturer#3 1671.68 NULL
+Manufacturer#1 1753.76 NULL
Manufacturer#2 1690.68 NULL
Manufacturer#2 1698.66 NULL
Manufacturer#2 1701.6 NULL
-Manufacturer#1 1753.76 NULL
-Manufacturer#5 1788.73 NULL
-Manufacturer#5 1789.69 NULL
Manufacturer#2 1800.7 NULL
-Manufacturer#4 1844.92 NULL
-Manufacturer#3 1922.98 NULL
Manufacturer#2 2031.98 NULL
+Manufacturer#3 1190.27 NULL
+Manufacturer#3 1337.29 NULL
+Manufacturer#3 1410.39 NULL
+Manufacturer#3 1671.68 NULL
+Manufacturer#3 1922.98 NULL
+Manufacturer#4 1206.26 NULL
+Manufacturer#4 1290.35 NULL
+Manufacturer#4 1375.42 NULL
+Manufacturer#4 1620.67 NULL
+Manufacturer#4 1844.92 NULL
+Manufacturer#5 1018.1 NULL
+Manufacturer#5 1464.48 NULL
+Manufacturer#5 1611.66 NULL
+Manufacturer#5 1788.73 NULL
+Manufacturer#5 1789.69 NULL
diff --git a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out
index 25068da..a3d6194 100644
--- a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out
+++ b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out
@@ -1,4 +1,6 @@
-PREHOOK: query: -- 1. testNoPTFNoWindowing
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. testNoPTFNoWindowing
select p_mfgr, p_name, p_size
from part
distribute by p_mfgr
@@ -6,7 +8,9 @@ sort by p_name
PREHOOK: type: QUERY
PREHOOK: Input: default@part
#### A masked pattern was here ####
-POSTHOOK: query: -- 1. testNoPTFNoWindowing
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+-- 1. testNoPTFNoWindowing
select p_mfgr, p_name, p_size
from part
distribute by p_mfgr
@@ -14,31 +18,31 @@ sort by p_name
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#5 almond antique blue firebrick mint 31
Manufacturer#1 almond antique burnished rose metallic 2
Manufacturer#1 almond antique burnished rose metallic 2
-Manufacturer#3 almond antique chartreuse khaki white 17
Manufacturer#1 almond antique chartreuse lavender yellow 34
-Manufacturer#3 almond antique forest lavender goldenrod 14
-Manufacturer#4 almond antique gainsboro frosted violet 10
-Manufacturer#5 almond antique medium spring khaki 6
-Manufacturer#3 almond antique metallic orange dim 19
-Manufacturer#3 almond antique misty red olive 1
-Manufacturer#3 almond antique olive coral navajo 45
Manufacturer#1 almond antique salmon chartreuse burlywood 6
-Manufacturer#5 almond antique sky peru orange 2
+Manufacturer#1 almond aquamarine burnished black steel 28
+Manufacturer#1 almond aquamarine pink moccasin thistle 42
Manufacturer#2 almond antique violet chocolate turquoise 14
-Manufacturer#4 almond antique violet mint lemon 39
Manufacturer#2 almond antique violet turquoise frosted 40
-Manufacturer#1 almond aquamarine burnished black steel 28
-Manufacturer#5 almond aquamarine dodger light gainsboro 46
-Manufacturer#4 almond aquamarine floral ivory bisque 27
Manufacturer#2 almond aquamarine midnight light salmon 2
-Manufacturer#1 almond aquamarine pink moccasin thistle 42
Manufacturer#2 almond aquamarine rose maroon antique 25
Manufacturer#2 almond aquamarine sandy cyan gainsboro 18
+Manufacturer#3 almond antique chartreuse khaki white 17
+Manufacturer#3 almond antique forest lavender goldenrod 14
+Manufacturer#3 almond antique metallic orange dim 19
+Manufacturer#3 almond antique misty red olive 1
+Manufacturer#3 almond antique olive coral navajo 45
+Manufacturer#4 almond antique gainsboro frosted violet 10
+Manufacturer#4 almond antique violet mint lemon 39
+Manufacturer#4 almond aquamarine floral ivory bisque 27
Manufacturer#4 almond aquamarine yellow dodger mint 7
Manufacturer#4 almond azure aquamarine papaya violet 12
+Manufacturer#5 almond antique blue firebrick mint 31
+Manufacturer#5 almond antique medium spring khaki 6
+Manufacturer#5 almond antique sky peru orange 2
+Manufacturer#5 almond aquamarine dodger light gainsboro 46
Manufacturer#5 almond azure blanched chiffon midnight 23
PREHOOK: query: -- 2. testUDAFsNoWindowingNoPTFNoGBY
select p_mfgr,p_name, p_retailprice,
diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out
index a9574ee..33cfe84 100644
--- a/ql/src/test/results/clientpositive/spark/sample10.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample10.q.out
@@ -89,7 +89,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out
index 9d527ce..d90d50d 100644
--- a/ql/src/test/results/clientpositive/spark/semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out
@@ -158,7 +158,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -272,7 +272,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -388,7 +388,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -496,7 +496,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -615,7 +615,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -726,7 +726,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -837,7 +837,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -945,7 +945,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1058,7 +1058,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1185,7 +1185,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1318,7 +1318,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1447,7 +1447,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -1599,7 +1599,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1742,7 +1742,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -1894,7 +1894,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2002,8 +2002,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2185,7 +2185,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -2340,7 +2340,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (SORT, 1)
+ Reducer 3 <- Map 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2450,8 +2450,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2645,7 +2645,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (SORT, 1)
+ Reducer 4 <- Map 3 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 3
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
index 4cecdf2..b0f0706 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out
@@ -399,7 +399,7 @@ STAGE PLANS:
Spark
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -573,7 +573,7 @@ STAGE PLANS:
Spark
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
index 49405da..c35400a 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out
@@ -214,8 +214,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
- Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+ Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
index ecb8a65..1019c2e 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
@@ -225,7 +225,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out
index a09dff8..b9224ec 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out
@@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket_3
-PREHOOK: query: explain
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -219,10 +223,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 NULL NULL
3 val_3 NULL NULL
4 val_4 NULL NULL
5 val_5 NULL NULL
-10 val_10 NULL NULL
PREHOOK: query: explain
select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key
PREHOOK: type: QUERY
@@ -324,7 +328,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -388,10 +392,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 NULL NULL
3 val_3 NULL NULL
4 val_4 NULL NULL
5 val_5 NULL NULL
-10 val_10 NULL NULL
NULL NULL 20 val_20
NULL NULL 23 val_23
NULL NULL 25 val_25
@@ -569,10 +573,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 NULL NULL
3 val_3 NULL NULL
4 val_4 NULL NULL
5 val_5 NULL NULL
-10 val_10 NULL NULL
PREHOOK: query: explain
select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key
PREHOOK: type: QUERY
@@ -674,7 +678,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -738,10 +742,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 NULL NULL
3 val_3 NULL NULL
4 val_4 NULL NULL
5 val_5 NULL NULL
-10 val_10 NULL NULL
NULL NULL 20 val_20
NULL NULL 23 val_23
NULL NULL 25 val_25
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
index 4bbc455..fc4d7f0 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
@@ -208,7 +208,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -367,7 +367,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
index 49b9fda..6c736af 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
@@ -781,7 +781,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out
index e12a713..bf8b6de 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out
@@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket_3
-PREHOOK: query: explain
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -135,8 +139,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-4 val_4 4 val_4
10 val_10 10 val_10
+4 val_4 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
@@ -221,10 +225,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 10 val_10
3 val_3 NULL NULL
4 val_4 4 val_4
5 val_5 NULL NULL
-10 val_10 10 val_10
PREHOOK: query: explain
select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
@@ -308,8 +312,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-4 val_4 4 val_4
10 val_10 10 val_10
+4 val_4 4 val_4
NULL NULL 17 val_17
NULL NULL 19 val_19
NULL NULL 20 val_20
@@ -328,7 +332,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -392,10 +396,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 10 val_10
3 val_3 NULL NULL
4 val_4 4 val_4
5 val_5 NULL NULL
-10 val_10 10 val_10
NULL NULL 17 val_17
NULL NULL 19 val_19
NULL NULL 20 val_20
@@ -489,8 +493,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-4 val_4 4 val_4
10 val_10 10 val_10
+4 val_4 4 val_4
PREHOOK: query: explain
select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
@@ -575,10 +579,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 10 val_10
3 val_3 NULL NULL
4 val_4 4 val_4
5 val_5 NULL NULL
-10 val_10 10 val_10
PREHOOK: query: explain
select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key
PREHOOK: type: QUERY
@@ -662,8 +666,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-4 val_4 4 val_4
10 val_10 10 val_10
+4 val_4 4 val_4
NULL NULL 17 val_17
NULL NULL 19 val_19
NULL NULL 20 val_20
@@ -682,7 +686,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -746,10 +750,10 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL
+10 val_10 10 val_10
3 val_3 NULL NULL
4 val_4 4 val_4
5 val_5 NULL NULL
-10 val_10 10 val_10
NULL NULL 17 val_17
NULL NULL 19 val_19
NULL NULL 20 val_20
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out
index ca48bdd..3ce287f 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out
@@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket_3
-PREHOOK: query: explain
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -377,10 +381,10 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
PREHOOK: query: explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key right outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -488,12 +492,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
NULL NULL NULL NULL 20 val_20
NULL NULL NULL NULL 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -508,7 +512,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -587,16 +591,16 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
-NULL NULL NULL NULL 4 val_4
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
NULL NULL NULL NULL 20 val_20
NULL NULL NULL NULL 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -924,12 +928,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
+NULL NULL 20 val_20 20 val_20
+NULL NULL 23 val_23 23 val_23
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
-NULL NULL 20 val_20 20 val_20
-NULL NULL 23 val_23 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -944,7 +948,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1022,14 +1026,14 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
-NULL NULL NULL NULL 10 val_10
-NULL NULL NULL NULL 17 val_17
-NULL NULL NULL NULL 19 val_19
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
NULL NULL 30 val_30 NULL NULL
+NULL NULL NULL NULL 10 val_10
+NULL NULL NULL NULL 17 val_17
+NULL NULL NULL NULL 19 val_19
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -1044,7 +1048,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1138,7 +1142,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1217,10 +1221,10 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
@@ -1239,7 +1243,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1317,12 +1321,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
+NULL NULL 20 val_20 20 val_20
+NULL NULL 23 val_23 23 val_23
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
-NULL NULL 20 val_20 20 val_20
-NULL NULL 23 val_23 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -1337,7 +1341,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1416,15 +1420,15 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
-NULL NULL NULL NULL 4 val_4
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
-NULL NULL NULL NULL 10 val_10
-NULL NULL NULL NULL 17 val_17
-NULL NULL NULL NULL 19 val_19
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
NULL NULL 30 val_30 NULL NULL
+NULL NULL NULL NULL 10 val_10
+NULL NULL NULL NULL 17 val_17
+NULL NULL NULL NULL 19 val_19
+NULL NULL NULL NULL 4 val_4
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out
index ffb5c9b..82c42e4 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out
@@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket_3
-PREHOOK: query: explain
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -377,10 +381,10 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key right outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -488,12 +492,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
NULL NULL NULL NULL 20 val_20
NULL NULL NULL NULL 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -508,7 +512,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -587,16 +591,16 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
-NULL NULL NULL NULL 4 val_4
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
NULL NULL NULL NULL 20 val_20
NULL NULL NULL NULL 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -924,12 +928,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
+NULL NULL 20 val_20 20 val_20
+NULL NULL 23 val_23 23 val_23
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
-NULL NULL 20 val_20 20 val_20
-NULL NULL 23 val_23 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -944,7 +948,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1022,14 +1026,14 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
-NULL NULL NULL NULL 10 val_10
-NULL NULL NULL NULL 17 val_17
-NULL NULL NULL NULL 19 val_19
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
NULL NULL 30 val_30 NULL NULL
+NULL NULL NULL NULL 10 val_10
+NULL NULL NULL NULL 17 val_17
+NULL NULL NULL NULL 19 val_19
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -1044,7 +1048,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1138,7 +1142,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1217,10 +1221,10 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
@@ -1239,7 +1243,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1317,12 +1321,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
+NULL NULL 20 val_20 20 val_20
+NULL NULL 23 val_23 23 val_23
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
-NULL NULL 20 val_20 20 val_20
-NULL NULL 23 val_23 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -1337,7 +1341,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1416,15 +1420,15 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
-NULL NULL NULL NULL 4 val_4
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
-NULL NULL NULL NULL 10 val_10
-NULL NULL NULL NULL 17 val_17
-NULL NULL NULL NULL 19 val_19
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
NULL NULL 30 val_30 NULL NULL
+NULL NULL NULL NULL 10 val_10
+NULL NULL NULL NULL 17 val_17
+NULL NULL NULL NULL 19 val_19
+NULL NULL NULL NULL 4 val_4
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
index 363207a..b040a6a 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
@@ -623,7 +623,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out
index 802f8396..babaff8 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out
@@ -1,8 +1,12 @@
-PREHOOK: query: create table smb_bucket_input (key int, value string) stored as rcfile
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table smb_bucket_input (key int, value string) stored as rcfile
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@smb_bucket_input
-POSTHOOK: query: create table smb_bucket_input (key int, value string) stored as rcfile
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table smb_bucket_input (key int, value string) stored as rcfile
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@smb_bucket_input
@@ -68,11 +72,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
-4 val_356 NULL NULL
-NULL NULL 484 val_169
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
+4 val_356 NULL NULL
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
+NULL NULL 484 val_169
NULL NULL 5000 val_125
PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
PREHOOK: type: QUERY
@@ -84,11 +88,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
-4 val_356 NULL NULL
-NULL NULL 484 val_169
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
+4 val_356 NULL NULL
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
+NULL NULL 484 val_169
NULL NULL 5000 val_125
PREHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=2000 or key=4000
PREHOOK: type: QUERY
@@ -121,8 +125,8 @@ POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
NULL NULL 5000 val_125
PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
PREHOOK: type: QUERY
@@ -135,8 +139,8 @@ POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
NULL NULL 5000 val_125
PREHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=4000
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
index 4bb6290..552b5ae 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
@@ -281,7 +281,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -441,8 +441,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -609,7 +609,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -829,7 +829,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/union25.q.out b/ql/src/test/results/clientpositive/spark/union25.q.out
index abb93b6..b64ec53 100644
--- a/ql/src/test/results/clientpositive/spark/union25.q.out
+++ b/ql/src/test/results/clientpositive/spark/union25.q.out
@@ -68,8 +68,8 @@ STAGE PLANS:
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
Union 4 <- Map 7 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
- Reducer 5 <- Union 4 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
+ Reducer 5 <- Union 4 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out
index 0a81d4a..1fbc5b5 100644
--- a/ql/src/test/results/clientpositive/spark/union33.q.out
+++ b/ql/src/test/results/clientpositive/spark/union33.q.out
@@ -196,9 +196,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
index 41daaf6..d6d4add 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
@@ -276,8 +276,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -448,8 +448,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/uniquejoin.q.out b/ql/src/test/results/clientpositive/spark/uniquejoin.q.out
index b6d3553..b71d5b1 100644
--- a/ql/src/test/results/clientpositive/spark/uniquejoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/uniquejoin.q.out
@@ -46,14 +46,18 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@t3
-PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
SELECT a.key, b.key, c.key
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
PREHOOK: Input: default@t2
PREHOOK: Input: default@t3
#### A masked pattern was here ####
-POSTHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
SELECT a.key, b.key, c.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t1
@@ -63,14 +67,14 @@ POSTHOOK: Input: default@t3
1 NULL NULL
2 2 2
3 3 NULL
-NULL 4 4
-NULL 5 NULL
-NULL NULL 6
7 NULL 7
8 8 NULL
8 8 NULL
8 8 NULL
8 8 NULL
+NULL 4 4
+NULL 5 NULL
+NULL NULL 6
PREHOOK: query: FROM UNIQUEJOIN T1 a (a.key), T2 b (b.key), T3 c (c.key)
SELECT a.key, b.key, c.key
PREHOOK: type: QUERY
@@ -119,15 +123,15 @@ POSTHOOK: Input: default@t3
#### A masked pattern was here ####
1 11 NULL NULL NULL NULL
2 12 NULL NULL 2 12
-NULL NULL 2 22 NULL NULL
3 13 3 13 NULL NULL
-NULL NULL 4 14 4 14
-NULL NULL 5 15 NULL NULL
-NULL NULL NULL NULL 6 16
7 17 NULL NULL 7 17
8 18 8 18 NULL NULL
8 18 8 18 NULL NULL
8 28 NULL NULL NULL NULL
+NULL NULL 2 22 NULL NULL
+NULL NULL 4 14 4 14
+NULL NULL 5 15 NULL NULL
+NULL NULL NULL NULL 6 16
PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b (b.key), PRESERVE T3 c (c.key)
SELECT a.key, b.key, c.key
PREHOOK: type: QUERY
@@ -145,13 +149,13 @@ POSTHOOK: Input: default@t3
1 NULL NULL
2 2 2
3 3 NULL
-NULL 4 4
-NULL NULL 6
7 NULL 7
8 8 NULL
8 8 NULL
8 8 NULL
8 8 NULL
+NULL 4 4
+NULL NULL 6
PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b(b.key)
SELECT a.key, b.key
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
index 5ff382f..e43a0ca 100644
--- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
index 2fbe31f..c9ba3c89 100644
--- a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
@@ -16,7 +16,9 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@decimal_vgby
-PREHOOK: query: -- First only do simple aggregations that output primitives only
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+-- First only do simple aggregations that output primitives only
EXPLAIN SELECT cint,
COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1),
COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2)
@@ -24,7 +26,9 @@ EXPLAIN SELECT cint,
GROUP BY cint
HAVING COUNT(*) > 1
PREHOOK: type: QUERY
-POSTHOOK: query: -- First only do simple aggregations that output primitives only
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+-- First only do simple aggregations that output primitives only
EXPLAIN SELECT cint,
COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1),
COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2)
@@ -40,7 +44,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -113,13 +117,13 @@ POSTHOOK: query: SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby
#### A masked pattern was here ####
-762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925
-253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089
-3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 6 6984454.211097692 -4033.445769230769 6967702.8672438458471
-563 2 -515.621072973 -3367.6517567568 -3883.2728297298 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459
-6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462
-626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746
+253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089
528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003
+626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746
+6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462
+762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925
NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536
PREHOOK: query: -- Now add the others...
EXPLAIN SELECT cint,
@@ -145,7 +149,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -217,11 +221,11 @@ POSTHOOK: query: SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby
#### A masked pattern was here ####
-762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758
-253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274
-3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 1161283.811207307641183333 2604201.2704476737 2852759.5602156054
-563 2 -515.621072973 -3367.6517567568 -3883.2728297298 -1941.6364148649 1426.0153418918999 2016.6902366556308 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -2325.50327307692295 1707.9424961538462 2415.395441814127
-6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653
-626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603
+253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274
528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984
+626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603
+6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653
+762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758
NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 1956.576923076922966667 6821.495748565159 6822.606289190924
diff --git a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
index 7304a0c..279c61a 100644
--- a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
index 72cc00f..32bddd3 100644
--- a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
index b6a36d8..1927626 100644
--- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
@@ -257,7 +257,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
diff --git a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
index 6a7c990..0992573 100644
--- a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
@@ -115,7 +115,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
index 923d805..ce10280 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
@@ -28,6 +28,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
@@ -60,6 +61,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -71,6 +73,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -107,19 +110,31 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double)
+ outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
+ sort order: +++++
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -156,6 +171,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
@@ -189,50 +205,51 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL -31 1969-12-31 16:00:16.059 -31.0 NULL 31 -31 0 -31.0 -0.0 31.0 -2466.143 0.0 31.0 0.0 -10.175 -31.0 0.847741935483871 -31.0 0.0 -31
-true 50 1969-12-31 16:00:16.338 50.0 821UdmGbkEf4j -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50
-NULL 23 1969-12-31 16:00:11.785 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23
-true 14 1969-12-31 16:00:11.528 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14
-true 50 1969-12-31 16:00:11.087 50.0 1cGVWH7n1QU -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50
-NULL 13 1969-12-31 16:00:12.498 13.0 NULL -13 13 0 13.0 0.0 -13.0 1034.1889999999999 0.0 -13.0 0.0 -10.175 13.0 -2.0215384615384617 13.0 0.0 13
-true 36 1969-12-31 16:00:15.145 36.0 1cGVWH7n1QU -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36
-true 3 1969-12-31 16:00:14.349 3.0 cvLH6Eat2yFsyy7p -3 3 0 3.0 0.0 -3.0 238.659 0.0 -3.0 0.0 -10.175 3.0 -8.76 3.0 0.0 3
-NULL -4 1969-12-31 16:00:13.237 -4.0 NULL 4 -4 0 -4.0 -0.0 4.0 -318.212 0.0 4.0 0.0 -10.175 -4.0 6.57 -4.0 0.0 -4
-NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20
-true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30
-true -57 1969-12-31 16:00:13.365 -57.0 1cGVWH7n1QU 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
-true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40
-NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21
-NULL 7 1969-12-31 16:00:14.872 7.0 NULL -7 7 0 7.0 0.0 -7.0 556.871 0.0 -7.0 0.0 -10.175 7.0 -3.7542857142857144 7.0 0.0 7
-true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34
-NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
-true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
-NULL 44 1969-12-31 16:00:16.279 44.0 NULL -44 44 0 44.0 0.0 -44.0 3500.332 0.0 -44.0 0.0 -10.175 44.0 -0.5972727272727273 44.0 0.0 44
-true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
-NULL 0 1969-12-31 16:00:13.396 0.0 NULL 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0
-true -25 1969-12-31 16:00:15.516 -25.0 821UdmGbkEf4j 25 -25 0 -25.0 -0.0 25.0 -1988.8249999999998 0.0 25.0 0.0 -10.175 -25.0 1.0512000000000001 -25.0 0.0 -25
-true -40 1969-12-31 16:00:11.833 -40.0 cvLH6Eat2yFsyy7p 40 -40 0 -40.0 -0.0 40.0 -3182.12 0.0 40.0 0.0 -10.175 -40.0 0.657 -40.0 0.0 -40
-NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29
-NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
+NULL -64 1969-12-31 16:00:11.912 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:12.339 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:13.274 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -63 1969-12-31 16:00:11.946 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:12.188 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
NULL -63 1969-12-31 16:00:15.436 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
-NULL -6 1969-12-31 16:00:11.883 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6
-true 14 1969-12-31 16:00:11.302 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14
-true -63 1969-12-31 16:00:14.899 -63.0 cvLH6Eat2yFsyy7p 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
-true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13
-true 49 1969-12-31 16:00:15.541 49.0 cvLH6Eat2yFsyy7p -49 49 0 49.0 0.0 -49.0 3898.0969999999998 0.0 -49.0 0.0 -10.175 49.0 -0.5363265306122449 49.0 0.0 49
-true -60 1969-12-31 16:00:12.291 -60.0 821UdmGbkEf4j 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
-NULL 32 1969-12-31 16:00:14.026 32.0 NULL -32 32 0 32.0 0.0 -32.0 2545.696 0.0 -32.0 0.0 -10.175 32.0 -0.82125 32.0 0.0 32
-true 16 1969-12-31 16:00:15.698 16.0 1cGVWH7n1QU -16 16 0 16.0 0.0 -16.0 1272.848 0.0 -16.0 0.0 -10.175 16.0 -1.6425 16.0 0.0 16
-NULL 57 1969-12-31 16:00:14.011 57.0 NULL -57 57 0 57.0 0.0 -57.0 4534.521 0.0 -57.0 0.0 -10.175 57.0 -0.4610526315789474 57.0 0.0 57
-true 41 1969-12-31 16:00:12.235 41.0 cvLH6Eat2yFsyy7p -41 41 0 41.0 0.0 -41.0 3261.673 0.0 -41.0 0.0 -10.175 41.0 -0.6409756097560976 41.0 0.0 41
-true -64 1969-12-31 16:00:12.857 -64.0 821UdmGbkEf4j 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
-true 36 1969-12-31 16:00:13.825 36.0 cvLH6Eat2yFsyy7p -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36
-NULL -15 1969-12-31 16:00:15.186 -15.0 NULL 15 -15 0 -15.0 -0.0 15.0 -1193.295 0.0 15.0 0.0 -10.175 -15.0 1.752 -15.0 0.0 -15
-true -2 1969-12-31 16:00:15.431 -2.0 1cGVWH7n1QU 2 -2 0 -2.0 -0.0 2.0 -159.106 0.0 2.0 0.0 -10.175 -2.0 13.14 -2.0 0.0 -2
+NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:12.591 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.154 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.247 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.517 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.965 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -61 1969-12-31 16:00:11.842 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:12.454 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:14.192 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:16.558 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -60 1969-12-31 16:00:11.504 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -60 1969-12-31 16:00:11.641 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -60 1969-12-31 16:00:11.996 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -60 1969-12-31 16:00:12.779 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -59 1969-12-31 16:00:12.008 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:13.15 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:13.625 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:15.296 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:15.861 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -58 1969-12-31 16:00:12.065 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:12.683 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:12.948 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:14.066 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -57 1969-12-31 16:00:11.451 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:11.883 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:12.626 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:13.578 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:15.39 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -56 1969-12-31 16:00:13.602 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56
+NULL -56 1969-12-31 16:00:13.958 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56
+NULL -56 1969-12-31 16:00:15.038 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56
+NULL -55 1969-12-31 16:00:11.38 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
+NULL -55 1969-12-31 16:00:11.751 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
PREHOOK: query: -- double compare timestamp
EXPLAIN
SELECT cboolean1,
@@ -264,6 +281,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
POSTHOOK: query: -- double compare timestamp
@@ -297,6 +315,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -307,7 +326,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -344,19 +364,31 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double)
+ outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
+ sort order: +++++
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -393,6 +425,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
@@ -426,47 +459,48 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL -26 1969-12-31 16:00:08.33 -26.0 NULL 26 -26 0 -26.0 -0.0 26.0 -2068.3779999999997 0.0 26.0 0.0 -10.175 -26.0 1.0107692307692309 -26.0 0.0 -26
-NULL -6 1969-12-31 16:00:07.423 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6
-true 55 1969-12-31 15:59:58.969 55.0 cvLH6Eat2yFsyy7p -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55
-true 14 1969-12-31 16:00:08.209 14.0 1cGVWH7n1QU -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14
-true 48 1969-12-31 16:00:00.923 48.0 1cGVWH7n1QU -48 48 0 48.0 0.0 -48.0 3818.544 0.0 -48.0 0.0 -10.175 48.0 -0.5475 48.0 0.0 48
-true -10 1969-12-31 16:00:00.082 -10.0 1cGVWH7n1QU 10 -10 0 -10.0 -0.0 10.0 -795.53 0.0 10.0 0.0 -10.175 -10.0 2.628 -10.0 0.0 -10
-true -55 1969-12-31 16:00:05.591 -55.0 1cGVWH7n1QU 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
-NULL -35 1969-12-31 16:00:04.52 -35.0 NULL 35 -35 0 -35.0 -0.0 35.0 -2784.355 0.0 35.0 0.0 -10.175 -35.0 0.7508571428571429 -35.0 0.0 -35
-NULL -8 1969-12-31 16:00:00.65 -8.0 NULL 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8
-NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20
-true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30
-true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40
-NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21
-true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34
-NULL 23 1969-12-31 16:00:08.307 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23
-true 8 1969-12-31 16:00:10.496 8.0 821UdmGbkEf4j -8 8 0 8.0 0.0 -8.0 636.424 0.0 -8.0 0.0 -10.175 8.0 -3.285 8.0 0.0 8
-NULL -55 1969-12-31 16:00:08.896 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
-NULL -57 1969-12-31 16:00:02.707 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
-true 1 1969-12-31 16:00:00.125 1.0 cvLH6Eat2yFsyy7p -1 1 0 1.0 0.0 -1.0 79.553 0.0 -1.0 0.0 -10.175 1.0 -26.28 1.0 0.0 1
-true -1 1969-12-31 16:00:00.107 -1.0 1cGVWH7n1QU 1 -1 0 -1.0 -0.0 1.0 -79.553 0.0 1.0 0.0 -10.175 -1.0 26.28 -1.0 0.0 -1
-true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
-NULL 33 1969-12-31 16:00:07.006 33.0 NULL -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33
-NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
-true -6 1969-12-31 16:00:07.372 -6.0 cvLH6Eat2yFsyy7p 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6
-true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
-NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29
-NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
-true 33 1969-12-31 16:00:09.924 33.0 cvLH6Eat2yFsyy7p -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33
-NULL -18 1969-12-31 16:00:00.564 -18.0 NULL 18 -18 0 -18.0 -0.0 18.0 -1431.954 0.0 18.0 0.0 -10.175 -18.0 1.46 -18.0 0.0 -18
-NULL -51 1969-12-31 16:00:02.011 -51.0 NULL 51 -51 0 -51.0 -0.0 51.0 -4057.203 0.0 51.0 0.0 -10.175 -51.0 0.5152941176470588 -51.0 0.0 -51
-true -8 1969-12-31 16:00:02.109 -8.0 cvLH6Eat2yFsyy7p 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8
-true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13
-true -52 1969-12-31 16:00:01.824 -52.0 821UdmGbkEf4j 52 -52 0 -52.0 -0.0 52.0 -4136.755999999999 0.0 52.0 0.0 -10.175 -52.0 0.5053846153846154 -52.0 0.0 -52
-true -28 1969-12-31 16:00:10.554 -28.0 821UdmGbkEf4j 28 -28 0 -28.0 -0.0 28.0 -2227.484 0.0 28.0 0.0 -10.175 -28.0 0.9385714285714286 -28.0 0.0 -28
-true 2 1969-12-31 16:00:06.546 2.0 821UdmGbkEf4j -2 2 0 2.0 0.0 -2.0 159.106 0.0 -2.0 0.0 -10.175 2.0 -13.14 2.0 0.0 2
-true 40 1969-12-31 16:00:02.842 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40
-NULL 55 1969-12-31 16:00:02.215 55.0 NULL -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55
-NULL 22 1969-12-31 16:00:00.621 22.0 NULL -22 22 0 22.0 0.0 -22.0 1750.166 0.0 -22.0 0.0 -10.175 22.0 -1.1945454545454546 22.0 0.0 22
-NULL 58 1969-12-31 16:00:03.366 58.0 NULL -58 58 0 58.0 0.0 -58.0 4614.074 0.0 -58.0 0.0 -10.175 58.0 -0.4531034482758621 58.0 0.0 58
-true 0 1969-12-31 16:00:08.742 0.0 821UdmGbkEf4j 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0
+NULL -64 1969-12-31 16:00:00.199 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:00.29 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:01.785 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:03.944 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:05.997 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:10.858 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:11.912 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:12.339 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:13.274 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -63 1969-12-31 16:00:01.843 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:03.552 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:06.852 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:07.375 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:10.205 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:11.946 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:12.188 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:15.436 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -62 1969-12-31 16:00:00.037 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:01.22 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:01.515 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:01.734 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:02.373 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:03.85 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:08.198 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:09.025 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:09.889 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:10.069 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:10.225 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:10.485 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:12.591 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.154 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.247 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.517 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.965 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -61 1969-12-31 16:00:00.142 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:02.698 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:03.049 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:04.165 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:04.977 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index d02c96e..269c4e3 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@ -1866,7 +1866,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2074,7 +2074,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2329,7 +2329,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2662,7 +2662,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
index 1028345..5fe0020 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
@@ -228,8 +228,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -588,8 +588,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -839,7 +839,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1095,8 +1095,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1383,8 +1383,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1680,9 +1680,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2020,7 +2020,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2263,7 +2263,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -2589,8 +2589,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2866,8 +2866,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3147,8 +3147,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3435,9 +3435,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3762,8 +3762,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4222,8 +4222,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -4478,8 +4478,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4749,9 +4749,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -5222,11 +5222,11 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -5736,9 +5736,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6092,10 +6092,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6446,9 +6446,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6795,10 +6795,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+ Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -7194,9 +7194,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -7543,9 +7543,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
index 47c13d1..98f5ad2 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
@@ -64,7 +64,7 @@ static ActorSystemInfo createActorSystem(Map conf) throws IOExce
int akkaHeartBeatPauses = toInt(conf.get("spark.akka.heartbeat.pauses"), 600);
double akkaFailureDetector =
toDouble(conf.get("spark.akka.failure-detector.threshold"), 300.0);
- int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 1000);
+ int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 100);
// Disabled due to chill-akka depending on kryo 2.21, which is incompatible with 2.22
// due to packaging changes (relocated org.objenesis classes).
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
index 0aea4a2..903169d 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
@@ -17,10 +17,12 @@
package org.apache.hive.spark.client;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Serializable;
@@ -192,8 +194,12 @@ public void run() {
// If a Spark installation is provided, use the spark-submit script. Otherwise, call the
// SparkSubmit class directly, which has some caveats (like having to provide a proper
// version of Guava on the classpath depending on the deploy mode).
- if (conf.get("spark.home") != null) {
- argv.add(new File(conf.get("spark.home"), "bin/spark-submit").getAbsolutePath());
+ String sparkHome = conf.get("spark.home");
+ if (sparkHome == null) {
+ sparkHome = System.getProperty("spark.home");
+ }
+ if (sparkHome != null) {
+ argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath());
} else {
LOG.info("No spark.home provided, calling SparkSubmit directly.");
argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath());
@@ -248,12 +254,11 @@ public void run() {
LOG.debug("Running client driver with argv: {}", Joiner.on(" ").join(argv));
ProcessBuilder pb = new ProcessBuilder(argv.toArray(new String[argv.size()]));
- pb.environment().clear();
final Process child = pb.start();
int childId = childIdGenerator.incrementAndGet();
- redirect("stdout-redir-" + childId, child.getInputStream(), System.out);
- redirect("stderr-redir-" + childId, child.getErrorStream(), System.err);
+ redirect("stdout-redir-" + childId, child.getInputStream());
+ redirect("stderr-redir-" + childId, child.getErrorStream());
runnable = new Runnable() {
@Override
@@ -277,8 +282,8 @@ public void run() {
return thread;
}
- private void redirect(String name, InputStream in, OutputStream out) {
- Thread thread = new Thread(new Redirector(in, out));
+ private void redirect(String name, InputStream in) {
+ Thread thread = new Thread(new Redirector(in));
thread.setName(name);
thread.setDaemon(true);
thread.start();
@@ -345,23 +350,18 @@ public void onReceive(Object message) throws Exception {
private class Redirector implements Runnable {
- private final InputStream in;
- private final OutputStream out;
+ private final BufferedReader in;
- Redirector(InputStream in, OutputStream out) {
- this.in = in;
- this.out = out;
+ Redirector(InputStream in) {
+ this.in = new BufferedReader(new InputStreamReader(in));
}
@Override
public void run() {
try {
- byte[] buf = new byte[1024];
- int len = in.read(buf);
- while (len != -1) {
- out.write(buf, 0, len);
- out.flush();
- len = in.read(buf);
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ LOG.info(line);
}
} catch (Exception e) {
LOG.warn("Error in redirector thread.", e);