diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 8dcd4cf4432b261dff8d549b07704d75f4f1c769..0b9ea08ff605b183cd5063304569f34755a61618 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -595,4 +595,19 @@ public static boolean renameWithPerms(FileSystem fs, Path sourcePath, return false; } } + + /** + * @param fs1 + * @param fs2 + * @return return true if both file system arguments point to same file system + */ + public static boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) { + //When file system cache is disabled, you get different FileSystem objects + // for same file system, so '==' can't be used in such cases + //FileSystem api doesn't have a .equals() function implemented, so using + //the uri for comparison. FileSystem already uses uri+Configuration for + //equality in its CACHE . + //Once equality has been added in HDFS-4321, we should make use of it + return fs1.getUri().equals(fs2.getUri()); + } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 3126880aa1a1c5ed93c9576e84c980511e86145b..b7b92808194de4463b5c5d74093a513248ed689a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Database; @@ -156,7 +157,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, destPath = new Path(newTblLoc); destFs = wh.getFs(destPath); // check that src and dest are on the same file system - if (! equalsFileSystem(srcFs, destFs)) { + if (! FileUtils.equalsFileSystem(srcFs, destFs)) { throw new InvalidOperationException("table new location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported"); @@ -251,21 +252,6 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, } } - /** - * @param fs1 - * @param fs2 - * @return return true if both file system arguments point to same file system - */ - private boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) { - //When file system cache is disabled, you get different FileSystem objects - // for same file system, so '==' can't be used in such cases - //FileSystem api doesn't have a .equals() function implemented, so using - //the uri for comparison. FileSystem already uses uri+Configuration for - //equality in its CACHE . - //Once equality has been added in HDFS-4321, we should make use of it - return fs1.getUri().equals(fs2.getUri()); - } - public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List part_vals, final Partition new_part) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index a7e50adf02e795d192a29d918e3ac24665461e06..d079191ef4e21c13c9f6966ecf4d15657b056bbe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1241,7 +1241,7 @@ public void loadPartition(Path loadPath, String tableName, */ FileSystem oldPartPathFS = oldPartPath.getFileSystem(getConf()); FileSystem loadPathFS = loadPath.getFileSystem(getConf()); - if (oldPartPathFS.equals(loadPathFS)) { + if (FileUtils.equalsFileSystem(oldPartPathFS,loadPathFS)) { newPartPath = oldPartPath; } } diff --git a/ql/src/test/queries/clientpositive/alter_merge_3.q b/ql/src/test/queries/clientpositive/alter_merge_3.q new file mode 100644 index 0000000000000000000000000000000000000000..8d835ffc4562fa4ebf72f9a89393226a37fe31ee --- /dev/null +++ b/ql/src/test/queries/clientpositive/alter_merge_3.q @@ -0,0 +1,23 @@ +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/alter_merge_location/ds20140804; +dfs -put ../../data/files/smbbucket_1.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804; +dfs -put ../../data/files/smbbucket_2.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804; +dfs -put ../../data/files/smbbucket_3.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804; + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/alter_merge_location/ds20140805; +dfs -put ../../data/files/smbbucket_1.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805; +dfs -put ../../data/files/smbbucket_2.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805; +dfs -put ../../data/files/smbbucket_3.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805; + +create table src_rc_merge_test_part (key int, value string) partitioned by (ds string) stored as rcfile; + +set fs.hdfs.impl.disable.cache=false; +alter table src_rc_merge_test_part add partition (ds = '2014-08-04') location '${system:test.tmp.dir}/alter_merge_location/ds20140804'; +alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate; +select * from src_rc_merge_test_part where ds='2014-08-04'; + +set fs.hdfs.impl.disable.cache=true; +alter table src_rc_merge_test_part add partition (ds = '2014-08-05') location '${system:test.tmp.dir}/alter_merge_location/ds20140805'; +alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate; +select * from src_rc_merge_test_part where ds='2014-08-05'; + +drop table src_rc_merge_test_part; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/alter_merge_3.q.out b/ql/src/test/results/clientpositive/alter_merge_3.q.out new file mode 100644 index 0000000000000000000000000000000000000000..d820d2561a86bba34ef5dd51e4c7094e436878d3 --- /dev/null +++ b/ql/src/test/results/clientpositive/alter_merge_3.q.out @@ -0,0 +1,99 @@ +PREHOOK: query: create table src_rc_merge_test_part (key int, value string) partitioned by (ds string) stored as rcfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_rc_merge_test_part (key int, value string) partitioned by (ds string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_rc_merge_test_part +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@src_rc_merge_test_part +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@src_rc_merge_test_part +POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04 +PREHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_rc_merge_test_part +PREHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04 +POSTHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_rc_merge_test_part +POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04 +PREHOOK: query: select * from src_rc_merge_test_part where ds='2014-08-04' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_merge_test_part +PREHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-04 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_rc_merge_test_part where ds='2014-08-04' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_merge_test_part +POSTHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-04 +#### A masked pattern was here #### +1 val_1 2014-08-04 +3 val_3 2014-08-04 +4 val_4 2014-08-04 +5 val_5 2014-08-04 +10 val_10 2014-08-04 +20 val_20 2014-08-04 +23 val_23 2014-08-04 +25 val_25 2014-08-04 +30 val_30 2014-08-04 +4 val_4 2014-08-04 +10 val_10 2014-08-04 +17 val_17 2014-08-04 +19 val_19 2014-08-04 +20 val_20 2014-08-04 +23 val_23 2014-08-04 +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@src_rc_merge_test_part +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@src_rc_merge_test_part +POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05 +PREHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_rc_merge_test_part +PREHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05 +POSTHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_rc_merge_test_part +POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05 +PREHOOK: query: select * from src_rc_merge_test_part where ds='2014-08-05' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_merge_test_part +PREHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-05 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_rc_merge_test_part where ds='2014-08-05' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_merge_test_part +POSTHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-05 +#### A masked pattern was here #### +1 val_1 2014-08-05 +3 val_3 2014-08-05 +4 val_4 2014-08-05 +5 val_5 2014-08-05 +10 val_10 2014-08-05 +20 val_20 2014-08-05 +23 val_23 2014-08-05 +25 val_25 2014-08-05 +30 val_30 2014-08-05 +4 val_4 2014-08-05 +10 val_10 2014-08-05 +17 val_17 2014-08-05 +19 val_19 2014-08-05 +20 val_20 2014-08-05 +23 val_23 2014-08-05 +PREHOOK: query: drop table src_rc_merge_test_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_rc_merge_test_part +PREHOOK: Output: default@src_rc_merge_test_part +POSTHOOK: query: drop table src_rc_merge_test_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_rc_merge_test_part +POSTHOOK: Output: default@src_rc_merge_test_part