commit a523a8715b3919f0d8a45e0fad4efc13fe350d63 Author: Vihang Karajgaonkar Date: Tue Aug 23 17:14:19 2016 -0700 HIVE-14614 : Insert overwrite local directory fails with IllegalStateException diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 3785b1e2bb661d58548e6237ffb0156b598c793f..4667f68eb273980588ff86e15b90e034b5d0fe8c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.common.BlobStorageUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.TaskRunner; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.Heartbeater; @@ -361,7 +362,9 @@ public Path getMRScratchDir() { * @return A path to the new temporary directory */ public Path getTempDirForPath(Path path) { - if (BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf)) { + boolean isLocal = isPathLocal(path); + if ((BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf)) + || isLocal) { // For better write performance, we use HDFS for temporary data when object store is used. // Note that the scratch directory configuration variable must use HDFS or any other non-blobstorage system // to take advantage of this performance. @@ -371,6 +374,20 @@ public Path getTempDirForPath(Path path) { } } + /* + * Checks if the path is for the local filesystem or not + */ + private boolean isPathLocal(Path path) { + boolean isLocal = false; + if (path != null) { + String scheme = path.toUri().getScheme(); + if (scheme != null) { + isLocal = scheme.equals(Utilities.HADOOP_LOCAL_FS_SCHEME); + } + } + return isLocal; + } + private Path getExternalScratchDir(URI extURI) { return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !explain); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index a542dc4a364e4dfcee145f95f6876f713dc0ef62..b4c6982c79977e3a3366188a9e725fc0c957061e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -213,6 +213,7 @@ */ public static String HADOOP_LOCAL_FS = "file:///"; + public static final String HADOOP_LOCAL_FS_SCHEME = "file"; public static String MAP_PLAN_NAME = "map.xml"; public static String REDUCE_PLAN_NAME = "reduce.xml"; public static String MERGE_PLAN_NAME = "merge.xml";