diff --git data/conf/spark/yarn-client/hive-site.xml data/conf/spark/yarn-client/hive-site.xml
index 9cda40d..7d6219a 100644
--- data/conf/spark/yarn-client/hive-site.xml
+++ data/conf/spark/yarn-client/hive-site.xml
@@ -260,6 +260,11 @@
+ yarn.scheduler.increment-allocation-mb
+ 512
+
+
+
spark.testing
true
diff --git ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
index be33189..399d929 100644
--- ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
+++ ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
@@ -59,7 +59,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -155,8 +155,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -287,8 +287,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -419,7 +419,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -508,7 +508,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -599,7 +599,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -688,7 +688,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -779,7 +779,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/index_bitmap3.q.out ql/src/test/results/clientpositive/spark/index_bitmap3.q.out
index 31c921c..46e872d 100644
--- ql/src/test/results/clientpositive/spark/index_bitmap3.q.out
+++ ql/src/test/results/clientpositive/spark/index_bitmap3.q.out
@@ -106,8 +106,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out
index e3b205c..8befcc5 100644
--- ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out
+++ ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out
@@ -117,8 +117,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
index 356f625..5151b17 100644
--- ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
+++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
@@ -182,7 +182,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -311,7 +311,7 @@ Table: test_table_out
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
- numFiles 2
+ numFiles 4
numRows 0
rawDataSize 0
totalSize 0
@@ -457,7 +457,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -563,7 +563,7 @@ Table: test_table_out
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
- numFiles 2
+ numFiles 4
numRows 309
rawDataSize 2728
totalSize 3037
diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out
index bf010e7..85f43af 100644
--- ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out
+++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out
@@ -296,7 +296,7 @@ Table: test_table
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
- numFiles 3
+ numFiles 4
numRows 5
rawDataSize 19
totalSize 24
diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
index d9da762..264795a 100644
--- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
+++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
@@ -48,7 +48,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -235,7 +235,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -738,7 +738,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -878,7 +878,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1047,8 +1047,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1194,8 +1194,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1396,7 +1396,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1502,7 +1502,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1758,7 +1758,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1907,7 +1907,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2047,7 +2047,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2261,7 +2261,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2410,7 +2410,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2529,7 +2529,7 @@ STAGE PLANS:
Edges:
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
Reducer 3 <- Reducer 2 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 2)
+ Reducer 5 <- Map 4 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2806,7 +2806,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2945,7 +2945,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3066,7 +3066,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3186,7 +3186,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3335,8 +3335,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3478,8 +3478,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3722,7 +3722,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Reducer 5 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
Reducer 5 <- Map 4 (GROUP, 1)
Reducer 7 <- Map 6 (GROUP, 1)
@@ -3990,8 +3990,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Reducer 5 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
Reducer 5 <- Map 4 (GROUP, 1)
Reducer 7 <- Map 6 (GROUP, 1)
#### A masked pattern was here ####
@@ -4289,8 +4289,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2), Reducer 9 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4), Reducer 9 (PARTITION-LEVEL SORT, 4)
Reducer 7 <- Map 6 (GROUP, 1)
Reducer 9 <- Map 8 (GROUP, 1)
#### A masked pattern was here ####
@@ -5403,7 +5403,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 2)
+ Reducer 4 <- Map 3 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -6341,8 +6341,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Reducer 5 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
Reducer 5 <- Map 4 (GROUP, 1)
Reducer 7 <- Map 6 (GROUP, 1)
#### A masked pattern was here ####
diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out
index 8a794e7..f28001e 100644
--- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out
+++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out
@@ -194,7 +194,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out
index 564a6a2..835b789 100644
--- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out
+++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out
@@ -87,7 +87,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -213,9 +213,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1000 Data size: 24624 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
+ Target column: hr (string)
partition key expr: hr
Statistics: Num rows: 1000 Data size: 24624 Basic stats: COMPLETE Column stats: NONE
- target column name: hr
target work: Map 1
Stage: Stage-3
@@ -249,9 +249,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
+ Target column: ds (string)
partition key expr: ds
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- target column name: ds
target work: Map 1
Local Work:
Map Reduce Local Work
@@ -387,9 +387,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
+ Target column: ds (string)
partition key expr: ds
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- target column name: ds
target work: Map 1
Local Work:
Map Reduce Local Work
diff --git ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out
index 9d162c9..8f69f8c 100644
--- ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out
+++ ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out
@@ -10,7 +10,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -68,7 +68,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out
index f92356e..0a3ba5f 100644
--- ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out
+++ ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out
@@ -16,7 +16,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -147,7 +147,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -419,7 +419,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2), Map 6 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4), Map 6 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1