diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 0b31c42..1300ab5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.util.StringUtils; +import org.apache.velocity.runtime.parser.node.GetExecutor; /** * Context for Semantic Analyzers. Usage: not reusable - construct a new one for @@ -365,6 +366,16 @@ public Path getExternalTmpPath(URI extURI) { } /** + * This is similar to getExternalTmpPath() with difference being this method returns temp path + * within passed in uri, whereas getExternalTmpPath() ignores passed in path and returns temp + * path within /tmp + */ + public Path getExtTmpPathRelTo(URI uri) { + return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain, + uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId()); + } + + /** * @return the resFile */ public Path getResFile() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index ac9df5e..45556ca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1692,7 +1692,12 @@ public static Path createMoveTask(Task currTask, boolean // generate the temporary file // it must be on the same file system as the current destination Context baseCtx = parseCtx.getContext(); - Path tmpDir = baseCtx.getExternalTmpPath(dest.toUri()); + // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/.. + // to final location /user/hive/warehouse/ will fail later, so instead pick tmp dir + // on same namespace as tbl dir. + Path tmpDir = dest.toUri().getScheme().equals("viewfs") ? + baseCtx.getExtTmpPathRelTo(dest.getParent().toUri()) : + baseCtx.getExternalTmpPath(dest.toUri()); FileSinkDesc fileSinkDesc = fsOp.getConf(); // Change all the linked file sink descriptors diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 8e68fcf..fe64e46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -5332,7 +5332,12 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) if (isNonNativeTable) { queryTmpdir = dest_path; } else { - queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri()); + // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/.. + // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir + // on same namespace as tbl dir. + queryTmpdir = dest_path.toUri().getScheme().equals("viewfs") ? + ctx.getExtTmpPathRelTo(dest_path.getParent().toUri()) : + ctx.getExternalTmpPath(dest_path.toUri()); } if (dpCtx != null) { // set the root of the temporay path where dynamic partition columns will populate @@ -5425,7 +5430,12 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri() .getAuthority(), partPath.toUri().getPath()); - queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri()); + // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/.. + // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir + // on same namespace as tbl dir. + queryTmpdir = dest_path.toUri().getScheme().equals("viewfs") ? + ctx.getExtTmpPathRelTo(dest_path.getParent().toUri()) : + ctx.getExternalTmpPath(dest_path.toUri()); table_desc = Utilities.getTableDesc(dest_tab); // Add sorting/bucketing if needed