diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index bed621d3eb74f01e54110552f68538afd228018d..2851720310df579409f222a5bb32ebabe4a91049 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -12,6 +12,7 @@ minimr.query.files=auto_sortmerge_join_16.q,\ constprog_partitioner.q,\ disable_merge_for_bucketing.q,\ empty_dir_in_table.q,\ + exchgpartition2lel.q,\ external_table_with_space_in_location_path.q,\ file_with_header_footer.q,\ groupby2.q,\ diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 1840e76cc567e95e1942d912b8ab0db516d63a3b..8f222cb0958178d006a79285243281d16bdbdf58 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2545,6 +2545,27 @@ public Partition add_partition_with_environment_context( } return ret; } +//This method is used by exchange partition and assume that the destination table root(the tableRootPath) +//should exist +//For some file system does not allow create child folder if parent folder does not exist, this +//method make sure create parent folder first. + private void createFullPath(Path fullPath, Path tableRootPath, FileSystem fs) throws MetaException { + if(fullPath != null && fullPath.compareTo(tableRootPath) != 0) { + try { + if(fs.exists(fullPath)) + return; + Path parentPath = fullPath.getParent(); + if(!fs.exists(parentPath)) + createFullPath(parentPath,tableRootPath, fs); + if (!wh.mkdirs(fullPath, true)) { + throw new MetaException("Unable to create path " + fullPath); + } + } + catch(IOException ioe) { + throw new MetaException("Unable to create path " + fullPath); + } + } + } @Override public Partition exchange_partition(Map partitionSpecs, @@ -2596,6 +2617,10 @@ public Partition exchange_partition(Map partitionSpecs, ms.dropPartition(partition.getDbName(), sourceTable.getTableName(), partition.getValues()); } + Path destTableRoot = new Path(destinationTable.getSd().getLocation()); + FileSystem destFs = wh.getFs(destPath); + Path destParentPath = destPath.getParent(); + createFullPath(destParentPath, destTableRoot, destFs); /** * TODO: Use the hard link feature of hdfs * once https://issues.apache.org/jira/browse/HDFS-3370 is done diff --git a/ql/src/test/queries/clientpositive/exchgpartition2lel.q b/ql/src/test/queries/clientpositive/exchgpartition2lel.q new file mode 100644 index 0000000000000000000000000000000000000000..0b01a38647d719f59197ad56295eaba93b842756 --- /dev/null +++ b/ql/src/test/queries/clientpositive/exchgpartition2lel.q @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; + +CREATE TABLE t1 (a int) PARTITIONED BY (d1 int); +CREATE TABLE t2 (a int) PARTITIONED BY (d1 int); +CREATE TABLE t3 (a int) PARTITIONED BY (d1 int, d2 int); +CREATE TABLE t4 (a int) PARTITIONED BY (d1 int, d2 int); + +INSERT OVERWRITE TABLE t1 PARTITION (d1 = 1) SELECT key FROM src where key = 100 limit 1; +INSERT OVERWRITE TABLE t3 PARTITION (d1 = 1, d2 = 1) SELECT key FROM src where key = 100 limit 1; + +SELECT * FROM t1; + +SELECT * FROM t3; + +ALTER TABLE t2 EXCHANGE PARTITION (d1 = 1) WITH TABLE t1; +SELECT * FROM t1; +SELECT * FROM t2; + +ALTER TABLE t4 EXCHANGE PARTITION (d1 = 1, d2 = 1) WITH TABLE t3; +SELECT * FROM t3; +SELECT * FROM t4; + diff --git a/ql/src/test/results/clientpositive/exchgpartition2lel.q.out b/ql/src/test/results/clientpositive/exchgpartition2lel.q.out new file mode 100644 index 0000000000000000000000000000000000000000..3c97c44000f078a58e92580c0a7466c367309276 --- /dev/null +++ b/ql/src/test/results/clientpositive/exchgpartition2lel.q.out @@ -0,0 +1,134 @@ +PREHOOK: query: DROP TABLE IF EXISTS t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS t2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS t3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t3 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS t4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE t1 (a int) PARTITIONED BY (d1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1 (a int) PARTITIONED BY (d1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: CREATE TABLE t2 (a int) PARTITIONED BY (d1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: CREATE TABLE t2 (a int) PARTITIONED BY (d1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: CREATE TABLE t3 (a int) PARTITIONED BY (d1 int, d2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t3 +POSTHOOK: query: CREATE TABLE t3 (a int) PARTITIONED BY (d1 int, d2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t3 +PREHOOK: query: CREATE TABLE t4 (a int) PARTITIONED BY (d1 int, d2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t4 +POSTHOOK: query: CREATE TABLE t4 (a int) PARTITIONED BY (d1 int, d2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t4 +PREHOOK: query: INSERT OVERWRITE TABLE t1 PARTITION (d1 = 1) SELECT key FROM src where key = 100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1@d1=1 +POSTHOOK: query: INSERT OVERWRITE TABLE t1 PARTITION (d1 = 1) SELECT key FROM src where key = 100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1@d1=1 +POSTHOOK: Lineage: t1 PARTITION(d1=1).a EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE t3 PARTITION (d1 = 1, d2 = 1) SELECT key FROM src where key = 100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t3@d1=1/d2=1 +POSTHOOK: query: INSERT OVERWRITE TABLE t3 PARTITION (d1 = 1, d2 = 1) SELECT key FROM src where key = 100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t3@d1=1/d2=1 +POSTHOOK: Lineage: t3 PARTITION(d1=1,d2=1).a EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: SELECT * FROM t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t1@d1=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t1@d1=1 +#### A masked pattern was here #### +100 1 +PREHOOK: query: SELECT * FROM t3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +PREHOOK: Input: default@t3@d1=1/d2=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +POSTHOOK: Input: default@t3@d1=1/d2=1 +#### A masked pattern was here #### +100 1 1 +PREHOOK: query: ALTER TABLE t2 EXCHANGE PARTITION (d1 = 1) WITH TABLE t1 +PREHOOK: type: ALTERTABLE_EXCHANGEPARTITION +POSTHOOK: query: ALTER TABLE t2 EXCHANGE PARTITION (d1 = 1) WITH TABLE t1 +POSTHOOK: type: ALTERTABLE_EXCHANGEPARTITION +PREHOOK: query: SELECT * FROM t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM t2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t2@d1=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t2@d1=1 +#### A masked pattern was here #### +100 1 +PREHOOK: query: ALTER TABLE t4 EXCHANGE PARTITION (d1 = 1, d2 = 1) WITH TABLE t3 +PREHOOK: type: ALTERTABLE_EXCHANGEPARTITION +POSTHOOK: query: ALTER TABLE t4 EXCHANGE PARTITION (d1 = 1, d2 = 1) WITH TABLE t3 +POSTHOOK: type: ALTERTABLE_EXCHANGEPARTITION +PREHOOK: query: SELECT * FROM t3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM t4 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +PREHOOK: Input: default@t4@d1=1/d2=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t4 +POSTHOOK: Input: default@t4@d1=1/d2=1 +#### A masked pattern was here #### +100 1 1