diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index dd5bef80e2..52c2dd87cd 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -388,6 +388,8 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vectorized_timestamp_ints_casts.q minillap.query.files=acid_bucket_pruning.q,\ + alter_table_location2.q,\ + alter_table_location3.q,\ bucket5.q,\ bucket6.q,\ dynamic_semijoin_user_level.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index ee5e3907f5..6d35391333 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1979,7 +1979,7 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(), isSrcLocal, isAutoPurge, newFiles, FileUtils.HIDDEN_FILES_PATH_FILTER, needRecycle, isManaged); } else { - FileSystem fs = tbl.getDataLocation().getFileSystem(conf); + FileSystem fs = destPath.getFileSystem(conf); copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation, (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles, tbl.getNumBuckets() > 0, isFullAcidTable, isManaged); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e1036372bc..2ef2347473 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7342,10 +7342,9 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) checkImmutableTable(qb, destinationTable, partPath, true); - // if the table is in a different dfs than the partition, - // replace the partition's dfs with the table's dfs. - destinationPath = new Path(tabPath.toUri().getScheme(), tabPath.toUri() - .getAuthority(), partPath.toUri().getPath()); + // Previous behavior (HIVE-1707) used to replace the partition's dfs with the table's dfs. + // The changes in HIVE-19891 appears to no longer support that behavior. + destinationPath = partPath; isMmTable = AcidUtils.isInsertOnlyTable(destinationTable.getParameters()); queryTmpdir = isMmTable ? destinationPath : ctx.getTempDirForFinalJobPath(destinationPath); diff --git a/ql/src/test/queries/clientpositive/alter_table_location2.q b/ql/src/test/queries/clientpositive/alter_table_location2.q new file mode 100644 index 0000000000..80089f9dc6 --- /dev/null +++ b/ql/src/test/queries/clientpositive/alter_table_location2.q @@ -0,0 +1,21 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE alter_table_location2 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS ORC + LOCATION 'pfile://${system:test.tmp.dir}/alter_table_location2' + TBLPROPERTIES ("transactional"="true"); +; + +INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (8,'Henry','CSE'); +ALTER TABLE alter_table_location2 ADD PARTITION (year=2017); + +ALTER TABLE alter_table_location2 SET LOCATION 'hdfs:///tmp/alter_table_location2'; + +INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (9,'Horace','CSE'); +INSERT INTO alter_table_location2 PARTITION (year=2017) VALUES (10,'Harris','CSE'); +INSERT INTO alter_table_location2 PARTITION (year=2018) VALUES (11,'Humphrey','CSE'); + +SELECT * from alter_table_location2 order by id; + diff --git a/ql/src/test/queries/clientpositive/alter_table_location3.q b/ql/src/test/queries/clientpositive/alter_table_location3.q new file mode 100644 index 0000000000..bb15d51013 --- /dev/null +++ b/ql/src/test/queries/clientpositive/alter_table_location3.q @@ -0,0 +1,16 @@ + +CREATE TABLE alter_table_location3 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS TEXTFILE + LOCATION 'pfile://${system:test.tmp.dir}/alter_table_location3' +; + +INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (8,'Henry','CSE'); +ALTER TABLE alter_table_location3 ADD PARTITION (year=2017); + +ALTER TABLE alter_table_location3 SET LOCATION 'hdfs:///tmp/alter_table_location3'; +INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (9,'Horace','CSE'); +INSERT INTO alter_table_location3 PARTITION (year=2017) VALUES (10,'Harris','CSE'); +INSERT INTO alter_table_location3 PARTITION (year=2018) VALUES (11,'Humphrey','CSE'); + +SELECT * from alter_table_location3 order by id; diff --git a/ql/src/test/results/clientpositive/llap/alter_table_location2.q.out b/ql/src/test/results/clientpositive/llap/alter_table_location2.q.out new file mode 100644 index 0000000000..e4eee1874b --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/alter_table_location2.q.out @@ -0,0 +1,97 @@ +PREHOOK: query: CREATE TABLE alter_table_location2 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS ORC +#### A masked pattern was here #### + TBLPROPERTIES ("transactional"="true") +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@alter_table_location2 +POSTHOOK: query: CREATE TABLE alter_table_location2 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS ORC +#### A masked pattern was here #### + TBLPROPERTIES ("transactional"="true") +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter_table_location2 +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (8,'Henry','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (8,'Henry','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: ALTER TABLE alter_table_location2 ADD PARTITION (year=2017) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@alter_table_location2 +POSTHOOK: query: ALTER TABLE alter_table_location2 ADD PARTITION (year=2017) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@alter_table_location2 +POSTHOOK: Output: default@alter_table_location2@year=2017 +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_LOCATION +PREHOOK: Input: default@alter_table_location2 +PREHOOK: Output: default@alter_table_location2 +PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_LOCATION +POSTHOOK: Input: default@alter_table_location2 +POSTHOOK: Output: default@alter_table_location2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (9,'Horace','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (9,'Horace','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2017) VALUES (10,'Harris','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2017 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2017) VALUES (10,'Harris','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2017 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2017).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2017).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2017).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2018 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2018 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2018).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2018).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2018).name SCRIPT [] +PREHOOK: query: SELECT * from alter_table_location2 order by id +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_table_location2 +PREHOOK: Input: default@alter_table_location2@year=2016 +PREHOOK: Input: default@alter_table_location2@year=2017 +PREHOOK: Input: default@alter_table_location2@year=2018 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * from alter_table_location2 order by id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_table_location2 +POSTHOOK: Input: default@alter_table_location2@year=2016 +POSTHOOK: Input: default@alter_table_location2@year=2017 +POSTHOOK: Input: default@alter_table_location2@year=2018 +POSTHOOK: Output: hdfs://### HDFS PATH ### +8 Henry CSE 2016 +9 Horace CSE 2016 +10 Harris CSE 2017 +11 Humphrey CSE 2018 diff --git a/ql/src/test/results/clientpositive/llap/alter_table_location3.q.out b/ql/src/test/results/clientpositive/llap/alter_table_location3.q.out new file mode 100644 index 0000000000..d9db63112a --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/alter_table_location3.q.out @@ -0,0 +1,95 @@ +PREHOOK: query: CREATE TABLE alter_table_location3 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS TEXTFILE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@alter_table_location3 +POSTHOOK: query: CREATE TABLE alter_table_location3 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS TEXTFILE +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter_table_location3 +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (8,'Henry','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (8,'Henry','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: ALTER TABLE alter_table_location3 ADD PARTITION (year=2017) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@alter_table_location3 +POSTHOOK: query: ALTER TABLE alter_table_location3 ADD PARTITION (year=2017) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@alter_table_location3 +POSTHOOK: Output: default@alter_table_location3@year=2017 +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_LOCATION +PREHOOK: Input: default@alter_table_location3 +PREHOOK: Output: default@alter_table_location3 +PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_LOCATION +POSTHOOK: Input: default@alter_table_location3 +POSTHOOK: Output: default@alter_table_location3 +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (9,'Horace','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (9,'Horace','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2017) VALUES (10,'Harris','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2017 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2017) VALUES (10,'Harris','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2017 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2017).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2017).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2017).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2018 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2018 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2018).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2018).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2018).name SCRIPT [] +PREHOOK: query: SELECT * from alter_table_location3 order by id +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_table_location3 +PREHOOK: Input: default@alter_table_location3@year=2016 +PREHOOK: Input: default@alter_table_location3@year=2017 +PREHOOK: Input: default@alter_table_location3@year=2018 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * from alter_table_location3 order by id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_table_location3 +POSTHOOK: Input: default@alter_table_location3@year=2016 +POSTHOOK: Input: default@alter_table_location3@year=2017 +POSTHOOK: Input: default@alter_table_location3@year=2018 +POSTHOOK: Output: hdfs://### HDFS PATH ### +8 Henry CSE 2016 +9 Horace CSE 2016 +10 Harris CSE 2017 +11 Humphrey CSE 2018