diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index 5fb5fd308ace29191aeb5f0a12f0fa38ad631244..92178cb6a32393bd5e49d1123b4523bbad9b9987 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -418,7 +418,18 @@ public static Path createMoveTask(Task currTask, boolean chDir, // it must be on the same file system as the current destination Context baseCtx = parseCtx.getContext(); - Path tmpDir = baseCtx.getExternalTmpPath(dest); + Path tmpDir = null; + + // The dest path (output location of the final job) may be an -mr-1000X dir, in case all below are true: + // -target table location FS is not HDFS but either blob storage or local FS + // -HIVE_BLOBSTORE_USE_BLOBSTORE_AS_SCRATCHDIR is set to false (default) + // -HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED is false + // In such case we shouldn't request an external tmp dir as it will end up inside the mr temp dir + if (baseCtx.isMRTmpFileURI(dest.toUri().getPath())) { + tmpDir = baseCtx.getMRTmpPath(); + } else { + tmpDir = baseCtx.getExternalTmpPath(dest); + } // Change all the linked file sink descriptors if (fileSinkDesc.getLinkedFileSinkDesc() != null) {