diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index 6380774..6f364b7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -179,6 +179,10 @@ private SparkJobRef submit(final DriverContext driverContext, final SparkWork sp final HiveConf hiveConf = (HiveConf) ctx.getConf(); refreshLocalResources(sparkWork, hiveConf); final JobConf jobConf = new JobConf(hiveConf); + //spark.master will be used in multiple operators ,so make sure its set. + if (jobConf.get("spark.master") == null) { + jobConf.set("spark.master", this.hiveConf.get("spark.master")); + } // Create temporary scratch dir final Path emptyScratchDir = ctx.getMRTmpPath();