diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java index 7d7e1a6..058d91d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java @@ -46,7 +46,7 @@ protected static final transient Log LOG = LogFactory.getLog(HiveSparkClientFactory.class); private static final String SPARK_DEFAULT_CONF_FILE = "spark-defaults.conf"; - private static final String SPARK_DEFAULT_MASTER = "local"; + private static final String SPARK_DEFAULT_MASTER = "yarn-cluster"; private static final String SPARK_DEFAULT_APP_NAME = "Hive on Spark"; private static final String SPARK_DEFAULT_SERIALIZER = "org.apache.spark.serializer.KryoSerializer"; private static final String SPARK_DEFAULT_REFERENCE_TRACKING = "false"; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java index ad012b6..0b62e73 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java @@ -17,14 +17,11 @@ */ package org.apache.hadoop.hive.ql.exec.spark.session; -import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClientFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.shims.Utils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.spark.client.SparkClientFactory; import java.io.IOException; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java index e74ed87..a80c6f8 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java @@ -42,6 +42,7 @@ @Test public void testSingleSessionMultipleUse() throws Exception { HiveConf conf = new HiveConf(); + conf.set("spark.master", "local"); SparkSessionManager sessionManager = SparkSessionManagerImpl.getInstance(); SparkSession sparkSession1 = sessionManager.getSession(null, conf, true); @@ -68,6 +69,8 @@ public void testMultiSessionMultipleUse() throws Exception { sessionManagerHS2.shutdown(); HiveConf hiveConf = new HiveConf(); + hiveConf.set("spark.master", "local"); + sessionManagerHS2.setup(hiveConf); List threadList = new ArrayList(); @@ -105,6 +108,7 @@ public void run() { String threadName = Thread.currentThread().getName(); System.out.println(threadName + " started."); HiveConf conf = new HiveConf(); + conf.set("spark.master", "local"); SparkSession prevSession = null; SparkSession currentSession = null;