commit 34c954f3925847eddb00345234f5107b03b72c87 Author: Vihang Karajgaonkar Date: Tue Aug 23 17:14:19 2016 -0700 HIVE-14614 : Insert overwrite local directory fails with IllegalStateException diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 3785b1e2bb661d58548e6237ffb0156b598c793f..4667f68eb273980588ff86e15b90e034b5d0fe8c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.common.BlobStorageUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.TaskRunner; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.Heartbeater; @@ -361,7 +362,9 @@ public Path getMRScratchDir() { * @return A path to the new temporary directory */ public Path getTempDirForPath(Path path) { - if (BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf)) { + boolean isLocal = isPathLocal(path); + if ((BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf)) + || isLocal) { // For better write performance, we use HDFS for temporary data when object store is used. // Note that the scratch directory configuration variable must use HDFS or any other non-blobstorage system // to take advantage of this performance. @@ -371,6 +374,20 @@ public Path getTempDirForPath(Path path) { } } + /* + * Checks if the path is for the local filesystem or not + */ + private boolean isPathLocal(Path path) { + boolean isLocal = false; + if (path != null) { + String scheme = path.toUri().getScheme(); + if (scheme != null) { + isLocal = scheme.equals(Utilities.HADOOP_LOCAL_FS_SCHEME); + } + } + return isLocal; + } + private Path getExternalScratchDir(URI extURI) { return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !explain); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index a542dc4a364e4dfcee145f95f6876f713dc0ef62..b4c6982c79977e3a3366188a9e725fc0c957061e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -213,6 +213,7 @@ */ public static String HADOOP_LOCAL_FS = "file:///"; + public static final String HADOOP_LOCAL_FS_SCHEME = "file"; public static String MAP_PLAN_NAME = "map.xml"; public static String REDUCE_PLAN_NAME = "reduce.xml"; public static String MERGE_PLAN_NAME = "merge.xml"; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java index 4a4c2402cbeb3d82126691917690a0caa3e66f54..808cb947e11386c44e5e6f03abc35cdefd4d7675 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java @@ -54,6 +54,11 @@ public void testGetScratchDirectoriesForPaths() throws IOException { doReturn(mrTmpPath).when(spyContext).getMRTmpPath(); assertEquals(mrTmpPath, spyContext.getTempDirForPath(new Path("s3a://bucket/dir"))); + // When local filesystem paths are used, then getMRTmpPatch() should be called to + // get a temporary directory + assertEquals(mrTmpPath, spyContext.getTempDirForPath(new Path("file:/user"))); + assertEquals(mrTmpPath, spyContext.getTempDirForPath(new Path("file:///user"))); + // When Non-Object store paths are used, then getExtTmpPathRelTo is called to get a temporary // directory on the same path passed as a parameter Path tmpPathRelTo = new Path("hdfs://hostname/user");