diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index d9d4b372b0..7757717b63 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.io.NullScanFileSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -209,6 +210,10 @@ private SparkJobRef submit(final DriverContext driverContext, final SparkWork sp FileSystem fs = emptyScratchDir.getFileSystem(jobConf); fs.mkdirs(emptyScratchDir); + // make sure NullScanFileSystem can be loaded - HIVE-18442 + jobConf.set("fs." + NullScanFileSystem.getBaseScheme() + ".impl", + NullScanFileSystem.class.getCanonicalName()); + byte[] jobConfBytes = KryoSerializer.serializeJobConf(jobConf); byte[] scratchDirBytes = KryoSerializer.serialize(emptyScratchDir); byte[] sparkWorkBytes = KryoSerializer.serialize(sparkWork);