diff --git data/conf/spark/standalone/hive-site.xml data/conf/spark/standalone/hive-site.xml
index 989e65e..a3b52c7 100644
--- data/conf/spark/standalone/hive-site.xml
+++ data/conf/spark/standalone/hive-site.xml
@@ -201,7 +201,7 @@
spark.master
- local-cluster[2,2,1024]
+ local-cluster[1,2,1024]
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 825f826..f6f35d3 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1185,7 +1185,8 @@ public void setSparkSession(SparkSession sparkSession) {
try {
// Wait a little for cluster to init, at most 4 minutes
long endTime = System.currentTimeMillis() + 240000;
- while (sparkSession.getMemoryAndCores().getSecond() <= 1) {
+ int expectedCores = conf.getInt("spark.executor.instances", 1) * 2;
+ while (sparkSession.getMemoryAndCores().getSecond() < expectedCores) {
if (System.currentTimeMillis() >= endTime) {
String msg = "Timed out waiting for Spark cluster to init";
throw new IllegalStateException(msg);
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
index 8224ef9..54d2cec 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
@@ -90,7 +90,7 @@ public SparkJobRef submit(DriverContext driverContext, SparkWork sparkWork) thro
long totalMemory = (long) (numExecutors * executorMemoryInMB * memoryFraction * 1024 * 1024);
int totalCores;
String masterURL = sparkConf.get("spark.master");
- if (masterURL.startsWith("spark")) {
+ if (masterURL.startsWith("spark") || masterURL.startsWith("local")) {
totalCores = sparkConf.contains("spark.default.parallelism") ?
sparkConf.getInt("spark.default.parallelism", 1) :
hiveSparkClient.getDefaultParallelism();
diff --git ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
index be33189..399d929 100644
--- ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
+++ ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
@@ -59,7 +59,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -155,8 +155,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -287,8 +287,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -419,7 +419,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -508,7 +508,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -599,7 +599,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -688,7 +688,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -779,7 +779,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/index_bitmap3.q.out ql/src/test/results/clientpositive/spark/index_bitmap3.q.out
index 31c921c..46e872d 100644
--- ql/src/test/results/clientpositive/spark/index_bitmap3.q.out
+++ ql/src/test/results/clientpositive/spark/index_bitmap3.q.out
@@ -106,8 +106,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out
index e3b205c..8befcc5 100644
--- ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out
+++ ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out
@@ -117,8 +117,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
index 356f625..5151b17 100644
--- ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
+++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
@@ -182,7 +182,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -311,7 +311,7 @@ Table: test_table_out
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
- numFiles 2
+ numFiles 4
numRows 0
rawDataSize 0
totalSize 0
@@ -457,7 +457,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -563,7 +563,7 @@ Table: test_table_out
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
- numFiles 2
+ numFiles 4
numRows 309
rawDataSize 2728
totalSize 3037
diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out
index bf010e7..85f43af 100644
--- ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out
+++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out
@@ -296,7 +296,7 @@ Table: test_table
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
- numFiles 3
+ numFiles 4
numRows 5
rawDataSize 19
totalSize 24
diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
index d9da762..df615ac 100644
--- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
+++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
@@ -48,7 +48,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -235,7 +235,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -342,7 +342,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -491,7 +491,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -598,7 +598,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -711,10 +711,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean)
+ filterExpr: ((date = '2008-04-08') and abs(((- UDFToLong(concat(UDFToString(day(ds)), '0'))) + 10)) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((date = '2008-04-08') and ds is not null) (type: boolean)
+ predicate: ((date = '2008-04-08') and abs(((- UDFToLong(concat(UDFToString(day(ds)), '0'))) + 10)) is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ds (type: string)
@@ -738,7 +738,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -746,25 +746,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart
- filterExpr: ds is not null (type: boolean)
+ filterExpr: abs(((- UDFToLong(concat(UDFToString(day(ds)), '0'))) + 10)) is not null (type: boolean)
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ds (type: string)
- outputColumnNames: _col0
+ Filter Operator
+ predicate: abs(((- UDFToLong(concat(UDFToString(day(ds)), '0'))) + 10)) is not null (type: boolean)
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: abs(((- UDFToLong(concat(UDFToString(day(_col0)), '0'))) + 10)) (type: bigint)
- sort order: +
- Map-reduce partition columns: abs(((- UDFToLong(concat(UDFToString(day(_col0)), '0'))) + 10)) (type: bigint)
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: abs(((- UDFToLong(concat(UDFToString(day(_col0)), '0'))) + 10)) (type: bigint)
+ sort order: +
+ Map-reduce partition columns: abs(((- UDFToLong(concat(UDFToString(day(_col0)), '0'))) + 10)) (type: bigint)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean)
+ filterExpr: ((date = '2008-04-08') and abs(((- UDFToLong(concat(UDFToString(day(ds)), '0'))) + 10)) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((date = '2008-04-08') and ds is not null) (type: boolean)
+ predicate: ((date = '2008-04-08') and abs(((- UDFToLong(concat(UDFToString(day(ds)), '0'))) + 10)) is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ds (type: string)
@@ -878,7 +881,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1047,8 +1050,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1194,8 +1197,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1396,7 +1399,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1502,7 +1505,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1651,7 +1654,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1758,7 +1761,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -1907,7 +1910,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2047,7 +2050,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2154,7 +2157,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2261,7 +2264,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2410,7 +2413,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2529,7 +2532,7 @@ STAGE PLANS:
Edges:
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
Reducer 3 <- Reducer 2 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 2)
+ Reducer 5 <- Map 4 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2806,7 +2809,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -2945,7 +2948,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3066,7 +3069,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3186,7 +3189,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3335,8 +3338,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3478,8 +3481,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 5 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4)
Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -3722,7 +3725,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Reducer 5 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4)
Reducer 3 <- Reducer 2 (GROUP, 1)
Reducer 5 <- Map 4 (GROUP, 1)
Reducer 7 <- Map 6 (GROUP, 1)
@@ -3990,8 +3993,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Reducer 5 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
Reducer 5 <- Map 4 (GROUP, 1)
Reducer 7 <- Map 6 (GROUP, 1)
#### A masked pattern was here ####
@@ -4289,8 +4292,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2), Reducer 9 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 4), Reducer 2 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4), Reducer 9 (PARTITION-LEVEL SORT, 4)
Reducer 7 <- Map 6 (GROUP, 1)
Reducer 9 <- Map 8 (GROUP, 1)
#### A masked pattern was here ####
@@ -5403,7 +5406,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 2)
+ Reducer 4 <- Map 3 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 3
@@ -6341,8 +6344,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Reducer 5 (PARTITION-LEVEL SORT, 4), Reducer 7 (PARTITION-LEVEL SORT, 4)
+ Reducer 3 <- Reducer 2 (GROUP, 4)
Reducer 5 <- Map 4 (GROUP, 1)
Reducer 7 <- Map 6 (GROUP, 1)
#### A masked pattern was here ####
diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out
index 8a794e7..cc7819c 100644
--- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out
+++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_2.q.out
@@ -194,7 +194,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -348,7 +348,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -736,7 +736,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out
index 564a6a2..41e98a1 100644
--- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out
+++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_mapjoin_only.q.out
@@ -87,7 +87,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -213,9 +213,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1000 Data size: 24624 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
+ Target column: hr (string)
partition key expr: hr
Statistics: Num rows: 1000 Data size: 24624 Basic stats: COMPLETE Column stats: NONE
- target column name: hr
target work: Map 1
Stage: Stage-3
@@ -249,9 +249,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
+ Target column: ds (string)
partition key expr: ds
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- target column name: ds
target work: Map 1
Local Work:
Map Reduce Local Work
@@ -259,7 +259,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -387,9 +387,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
+ Target column: ds (string)
partition key expr: ds
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- target column name: ds
target work: Map 1
Local Work:
Map Reduce Local Work
@@ -397,7 +397,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out
index 9d162c9..8f69f8c 100644
--- ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out
+++ ql/src/test/results/clientpositive/spark/spark_explain_groupbyshuffle.q.out
@@ -10,7 +10,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -68,7 +68,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out
index f92356e..0a3ba5f 100644
--- ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out
+++ ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out
@@ -16,7 +16,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -147,7 +147,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -419,7 +419,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2), Map 6 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL SORT, 4), Map 6 (PARTITION-LEVEL SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index ae6f542..e9445eb 100644
--- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -95,6 +95,7 @@
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.test.MiniTezCluster;
@@ -493,6 +494,7 @@ public MiniSparkShim(Configuration conf, int numberOfTaskTrackers,
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false);
conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 2048);
configureImpersonation(conf);
mr.init(conf);