diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java index 0e3cefc..40ce4b4 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java @@ -216,7 +216,13 @@ public void externalTableReplicationWithCustomPaths() throws Throwable { DistributedFileSystem fs = primary.miniDFSCluster.getFileSystem(); fs.mkdirs(externalTableLocation, new FsPermission("777")); - List loadWithClause = externalTableBasePathWithClause(); + // Create base directory but use HDFS path without schema or authority details. + // Hive should pick up the local cluster's HDFS schema/authority. + externalTableBasePathWithClause(); + List loadWithClause = Collections.singletonList( + "'" + HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname + "'='" + + REPLICA_EXTERNAL_BASE + "'" + ); WarehouseInstance.Tuple bootstrapTuple = primary.run("use " + primaryDbName) .run("create external table a (i int, j int) " diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java index 012df9d..59b7c1c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -38,7 +39,6 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.StringWriter; -import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.HashSet; @@ -62,13 +62,21 @@ private ReplExternalTables(){} - public static String externalTableLocation(HiveConf hiveConf, String location) { - String currentPath = new Path(location).toUri().getPath(); + public static String externalTableLocation(HiveConf hiveConf, String location) throws SemanticException { String baseDir = hiveConf.get(HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname); - URI basePath = new Path(baseDir).toUri(); - String dataPath = currentPath.replaceFirst(Path.SEPARATOR, basePath.getPath() + Path.SEPARATOR); - Path dataLocation = new Path(basePath.getScheme(), basePath.getAuthority(), dataPath); - LOG.debug("incoming location: {} , new location: {}", location, dataLocation.toString()); + Path basePath = new Path(baseDir); + Path currentPath = new Path(location); + String targetPathWithoutSchemeAndAuth = basePath.toUri().getPath() + currentPath.toUri().getPath(); + Path dataLocation; + try { + dataLocation = PathBuilder.fullyQualifiedHDFSUri( + new Path(targetPathWithoutSchemeAndAuth), + basePath.getFileSystem(hiveConf) + ); + } catch (IOException e) { + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); + } + LOG.info("Incoming external table location: {} , new location: {}", location, dataLocation.toString()); return dataLocation.toString(); }