diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 46350a3..c9e9294 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1329,6 +1329,7 @@ miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\ infer_bucket_sort_num_buckets.q,\ infer_bucket_sort_reducers_power_two.q,\ input16_cc.q,\ + insert_overwrite_directory2.q,\ leftsemijoin_mr.q,\ list_bucket_dml_10.q,\ load_fs2.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 2b01fce..05bb4a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2874,7 +2874,7 @@ public static boolean moveFile(final HiveConf conf, Path srcf, final Path destf, final SessionState parentSession = SessionState.get(); if (isSrcLocal) { // For local src file, copy to hdfs - destFs.copyFromLocalFile(srcf, destf); + destFs.copyFromLocalFile(false, replace, srcf, destf); if (inheritPerms) { try { HdfsUtils.setFullFileStatus(conf, destStatus, destFs, destf, true); @@ -2893,6 +2893,21 @@ public static boolean moveFile(final HiveConf conf, Path srcf, final Path destf, conf); } else { if (destIsSubDir) { + if (replace) { + Path parent = new Path(getQualifiedPathWithoutSchemeAndAuthority(srcf, srcFs)); + Path fullDestPath = new Path( + getQualifiedPathWithoutSchemeAndAuthority(destf, destFs)); + while (!parent.getParent().equals(fullDestPath)) { + parent = parent.getParent(); + } + FileStatus[] existingFiles = destFs.listStatus( + destf, FileUtils.HIDDEN_FILES_PATH_FILTER); + for (FileStatus fileStatus : existingFiles) { + if (!fileStatus.getPath().getName().equals(parent.getName())) { + destFs.delete(fileStatus.getPath(), true); + } + } + } FileStatus[] srcs = destFs.listStatus(srcf, FileUtils.HIDDEN_FILES_PATH_FILTER); List> futures = new LinkedList<>(); diff --git a/ql/src/test/queries/clientpositive/insert_overwrite_directory2.q b/ql/src/test/queries/clientpositive/insert_overwrite_directory2.q new file mode 100644 index 0000000..dd337a9 --- /dev/null +++ b/ql/src/test/queries/clientpositive/insert_overwrite_directory2.q @@ -0,0 +1,17 @@ +create external table result(key string) location "${system:test.tmp.dir}/result"; + +set mapreduce.job.reduces=2; + +insert overwrite directory "${system:test.tmp.dir}/result" +select key from src group by key; + +select count(*) from result; + +set mapreduce.job.reduces=1; + +insert overwrite directory "${system:test.tmp.dir}/result" +select key from src group by key; + +select count(*) from result; + +drop table result; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/insert_overwrite_directory2.q.out b/ql/src/test/results/clientpositive/insert_overwrite_directory2.q.out new file mode 100644 index 0000000..c3232e7 --- /dev/null +++ b/ql/src/test/results/clientpositive/insert_overwrite_directory2.q.out @@ -0,0 +1,54 @@ +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@result +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@result +#### A masked pattern was here #### +select key from src group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +select key from src group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: select count(*) from result +PREHOOK: type: QUERY +PREHOOK: Input: default@result +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from result +POSTHOOK: type: QUERY +POSTHOOK: Input: default@result +#### A masked pattern was here #### +309 +#### A masked pattern was here #### +select key from src group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +select key from src group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: select count(*) from result +PREHOOK: type: QUERY +PREHOOK: Input: default@result +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from result +POSTHOOK: type: QUERY +POSTHOOK: Input: default@result +#### A masked pattern was here #### +309 +PREHOOK: query: drop table result +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@result +PREHOOK: Output: default@result +POSTHOOK: query: drop table result +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@result +POSTHOOK: Output: default@result diff --git a/ql/src/test/results/clientpositive/spark/insert_overwrite_directory2.q.out b/ql/src/test/results/clientpositive/spark/insert_overwrite_directory2.q.out new file mode 100644 index 0000000..c3232e7 --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/insert_overwrite_directory2.q.out @@ -0,0 +1,54 @@ +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@result +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@result +#### A masked pattern was here #### +select key from src group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +select key from src group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: select count(*) from result +PREHOOK: type: QUERY +PREHOOK: Input: default@result +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from result +POSTHOOK: type: QUERY +POSTHOOK: Input: default@result +#### A masked pattern was here #### +309 +#### A masked pattern was here #### +select key from src group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +select key from src group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: select count(*) from result +PREHOOK: type: QUERY +PREHOOK: Input: default@result +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from result +POSTHOOK: type: QUERY +POSTHOOK: Input: default@result +#### A masked pattern was here #### +309 +PREHOOK: query: drop table result +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@result +PREHOOK: Output: default@result +POSTHOOK: query: drop table result +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@result +POSTHOOK: Output: default@result