diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index b677d467d6..85f0a3b013 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4272,14 +4272,15 @@ public static void handleMmTableFinalPath(Path specPath, String unionSuffix, Con } } - HashSet committed = new HashSet<>(); + HashSet committed = new HashSet<>(); for (Path mfp : manifests) { try (FSDataInputStream mdis = fs.open(mfp)) { int fileCount = mdis.readInt(); for (int i = 0; i < fileCount; ++i) { String nextFile = mdis.readUTF(); Utilities.FILE_OP_LOGGER.trace("Looking at committed file: {}", nextFile); - if (!committed.add(nextFile)) { + Path path = fs.makeQualified(new Path(nextFile)); + if (!committed.add(path)) { throw new HiveException(nextFile + " was specified in multiple manifests"); } } @@ -4340,7 +4341,7 @@ public PathOnlyFileStatus(Path path) { } private static void cleanMmDirectory(Path dir, FileSystem fs, String unionSuffix, - int lbLevels, HashSet committed) throws IOException, HiveException { + int lbLevels, HashSet committed) throws IOException, HiveException { for (FileStatus child : fs.listStatus(dir)) { Path childPath = child.getPath(); if (lbLevels > 0) { @@ -4352,7 +4353,7 @@ private static void cleanMmDirectory(Path dir, FileSystem fs, String unionSuffix "Recursion into LB directory {}; levels remaining ", childPath, lbLevels - 1); cleanMmDirectory(childPath, fs, unionSuffix, lbLevels - 1, committed); } else { - if (committed.contains(childPath.toString())) { + if (committed.contains(childPath)) { throw new HiveException("LB FSOP has commited " + childPath + " outside of LB directory levels " + lbLevels); } @@ -4362,12 +4363,12 @@ private static void cleanMmDirectory(Path dir, FileSystem fs, String unionSuffix } // No more LB directories expected. if (unionSuffix == null) { - if (committed.remove(childPath.toString())) { + if (committed.remove(childPath)) { continue; // A good file. } deleteUncommitedFile(childPath, fs); } else if (!child.isDirectory()) { - if (committed.contains(childPath.toString())) { + if (committed.contains(childPath)) { throw new HiveException("Union FSOP has commited " + childPath + " outside of union directory " + unionSuffix); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index 0a535d15c1..c23fd2c8b0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -236,8 +236,10 @@ public static String relativeToAbsolutePath(HiveConf conf, String location) } return uri.toString(); } else { - // no-op for non-test mode for now return location; + /* TODO: this should always make the path qualified to avoid ambiguity + Path path = new Path(location); + return path.getFileSystem(conf).makeQualified(path).toString();*/ } } catch (IOException e) { throw new SemanticException(ErrorMsg.IO_ERROR.getMsg() + ": " + e.getMessage(), e); diff --git ql/src/test/queries/clientpositive/mm_loc_ctas.q ql/src/test/queries/clientpositive/mm_loc_ctas.q new file mode 100644 index 0000000000..8e4cbbbf5e --- /dev/null +++ ql/src/test/queries/clientpositive/mm_loc_ctas.q @@ -0,0 +1,19 @@ +--! qt:dataset:src + +set hive.metastore.dml.events=true; +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +set hive.fetch.task.conversion=none; +set tez.grouping.min-size=1; +set tez.grouping.max-size=2; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + + +drop table test; +create table test(id int, name string); +insert into test values(1, 'aa'),(2,'bb'); + +drop table test3; +CREATE TABLE test3 stored as textfile LOCATION '${system:test.tmp.dir}/test2' tblproperties('transactional'='true', 'transactional_properties'='insert_only') AS SELECT * from test; \ No newline at end of file diff --git ql/src/test/results/clientpositive/mm_loc_ctas.q.out ql/src/test/results/clientpositive/mm_loc_ctas.q.out new file mode 100644 index 0000000000..471d8355f9 --- /dev/null +++ ql/src/test/results/clientpositive/mm_loc_ctas.q.out @@ -0,0 +1,38 @@ +PREHOOK: query: drop table test +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table test +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table test(id int, name string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test +POSTHOOK: query: create table test(id int, name string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test +PREHOOK: query: insert into test values(1, 'aa'),(2,'bb') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test +POSTHOOK: query: insert into test values(1, 'aa'),(2,'bb') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test +POSTHOOK: Lineage: test.id SCRIPT [] +POSTHOOK: Lineage: test.name SCRIPT [] +PREHOOK: query: drop table test3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table test3 +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@test +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@test3 +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@test +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test3