diff --git ql/src/test/queries/clientpositive/msck_repair_4.q ql/src/test/queries/clientpositive/msck_repair_4.q new file mode 100644 index 0000000000..aaf10a454d --- /dev/null +++ ql/src/test/queries/clientpositive/msck_repair_4.q @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS repairtable_n4; + +CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC; + +MSCK TABLE repairtable_n4; +show partitions repairtable_n4; + +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/Year=2020/Month=3/Day=1; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/Year=2020/Month=3/Day=2; + +MSCK TABLE repairtable_n4; +show partitions repairtable_n4; + +DROP TABLE default.repairtable_n4; diff --git ql/src/test/queries/clientpositive/msck_repair_5.q ql/src/test/queries/clientpositive/msck_repair_5.q new file mode 100644 index 0000000000..edb719893c --- /dev/null +++ ql/src/test/queries/clientpositive/msck_repair_5.q @@ -0,0 +1,13 @@ +CREATE EXTERNAL TABLE repairtable_n5(key INT, value STRING) PARTITIONED BY (Country String) stored as ORC; + +MSCK TABLE repairtable_n5; +show partitions repairtable_n5; + +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/Country=US; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/Country=us; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/Country=India; + +MSCK TABLE repairtable_n5; +show partitions repairtable_n5; + +DROP TABLE default.repairtable_n5; diff --git ql/src/test/queries/clientpositive/msck_repair_6.q ql/src/test/queries/clientpositive/msck_repair_6.q new file mode 100644 index 0000000000..ba449b8a01 --- /dev/null +++ ql/src/test/queries/clientpositive/msck_repair_6.q @@ -0,0 +1,11 @@ +CREATE EXTERNAL TABLE repairtable_n6(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC location '${system:test.tmp.dir}/apps/hive/warehouse/test.db/Repairtable_n6/'; + +MSCK TABLE repairtable_n6; +show partitions repairtable_n6; + +dfs ${system:test.dfs.mkdir} -p ${system:test.tmp.dir}/apps/hive/warehouse/test.db/Repairtable_n6/Year=2020/Month=4/Day=1; + +MSCK TABLE repairtable_n6; +show partitions repairtable_n6; + +DROP TABLE default.repairtable_n6; diff --git ql/src/test/results/clientpositive/msck_repair_4.q.out ql/src/test/results/clientpositive/msck_repair_4.q.out new file mode 100644 index 0000000000..2bbf38679d --- /dev/null +++ ql/src/test/results/clientpositive/msck_repair_4.q.out @@ -0,0 +1,45 @@ +PREHOOK: query: DROP TABLE IF EXISTS repairtable_n4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@repairtable_n4 +POSTHOOK: query: CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repairtable_n4 +PREHOOK: query: MSCK TABLE repairtable_n4 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n4 +POSTHOOK: query: MSCK TABLE repairtable_n4 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n4 +PREHOOK: query: show partitions repairtable_n4 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n4 +POSTHOOK: query: show partitions repairtable_n4 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n4 +PREHOOK: query: MSCK TABLE repairtable_n4 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n4 +POSTHOOK: query: MSCK TABLE repairtable_n4 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n4 +Partitions not in metastore: repairtable_n4:year=2020/month=3/day=1 repairtable_n4:year=2020/month=3/day=2 +PREHOOK: query: show partitions repairtable_n4 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n4 +POSTHOOK: query: show partitions repairtable_n4 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n4 +PREHOOK: query: DROP TABLE default.repairtable_n4 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@repairtable_n4 +PREHOOK: Output: default@repairtable_n4 +POSTHOOK: query: DROP TABLE default.repairtable_n4 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@repairtable_n4 +POSTHOOK: Output: default@repairtable_n4 diff --git ql/src/test/results/clientpositive/msck_repair_5.q.out ql/src/test/results/clientpositive/msck_repair_5.q.out new file mode 100644 index 0000000000..1b093e0fdf --- /dev/null +++ ql/src/test/results/clientpositive/msck_repair_5.q.out @@ -0,0 +1,41 @@ +PREHOOK: query: CREATE EXTERNAL TABLE repairtable_n5(key INT, value STRING) PARTITIONED BY (Country String) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@repairtable_n5 +POSTHOOK: query: CREATE EXTERNAL TABLE repairtable_n5(key INT, value STRING) PARTITIONED BY (Country String) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repairtable_n5 +PREHOOK: query: MSCK TABLE repairtable_n5 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n5 +POSTHOOK: query: MSCK TABLE repairtable_n5 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n5 +PREHOOK: query: show partitions repairtable_n5 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n5 +POSTHOOK: query: show partitions repairtable_n5 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n5 +PREHOOK: query: MSCK TABLE repairtable_n5 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n5 +POSTHOOK: query: MSCK TABLE repairtable_n5 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n5 +Partitions not in metastore: repairtable_n5:country=India repairtable_n5:country=US repairtable_n5:country=us +PREHOOK: query: show partitions repairtable_n5 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n5 +POSTHOOK: query: show partitions repairtable_n5 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n5 +PREHOOK: query: DROP TABLE default.repairtable_n5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@repairtable_n5 +PREHOOK: Output: default@repairtable_n5 +POSTHOOK: query: DROP TABLE default.repairtable_n5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@repairtable_n5 +POSTHOOK: Output: default@repairtable_n5 diff --git ql/src/test/results/clientpositive/msck_repair_6.q.out ql/src/test/results/clientpositive/msck_repair_6.q.out new file mode 100644 index 0000000000..004b966ddc --- /dev/null +++ ql/src/test/results/clientpositive/msck_repair_6.q.out @@ -0,0 +1,43 @@ +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@repairtable_n6 +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repairtable_n6 +PREHOOK: query: MSCK TABLE repairtable_n6 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n6 +POSTHOOK: query: MSCK TABLE repairtable_n6 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n6 +PREHOOK: query: show partitions repairtable_n6 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n6 +POSTHOOK: query: show partitions repairtable_n6 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n6 +PREHOOK: query: MSCK TABLE repairtable_n6 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n6 +POSTHOOK: query: MSCK TABLE repairtable_n6 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n6 +Partitions not in metastore: repairtable_n6:year=2020/month=4/day=1 +PREHOOK: query: show partitions repairtable_n6 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n6 +POSTHOOK: query: show partitions repairtable_n6 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n6 +PREHOOK: query: DROP TABLE default.repairtable_n6 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@repairtable_n6 +PREHOOK: Output: default@repairtable_n6 +POSTHOOK: query: DROP TABLE default.repairtable_n6 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@repairtable_n6 +POSTHOOK: Output: default@repairtable_n6 diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java index 5287f47e21..1137dbba26 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.MetaException; + +import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -108,6 +112,7 @@ public void setExpiredPartitions( public static class PartitionResult implements Comparable { private String partitionName; private String tableName; + private Path path; /** * @return name of partition @@ -139,6 +144,18 @@ public void setTableName(String tableName) { this.tableName = tableName; } + public void setPath(Path path) { + this.path = path; + } + + public Path getLocation(Path tablePath, Map partSpec) throws MetaException { + if (this.path == null) { + return new Path(tablePath, Warehouse.makePartPath(partSpec)); + } + + return this.path; + } + @Override public String toString() { return tableName + ":" + partitionName; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java index 6f4400a8ef..45af4e58c8 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java @@ -401,6 +401,10 @@ void findUnknownPartitions(Table table, Set partPaths, CheckResult.PartitionResult pr = new CheckResult.PartitionResult(); pr.setPartitionName(partitionName); pr.setTableName(table.getTableName()); + // Also set the correct partition path here as creating path from Warehouse.makePartPath will always return + // lowercase keys/path. Even if we add the new partition with lowerkeys, get queries on such partition + // will not return any results. + pr.setPath(partPath); result.getPartitionsNotInMs().add(pr); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java index f4e109d1b0..ab0c649f90 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java @@ -373,7 +373,7 @@ public Void execute(int size) throws MetastoreException { continue; } Map partSpec = Warehouse.makeSpecFromName(part.getPartitionName()); - Path location = new Path(tablePath, Warehouse.makePartPath(partSpec)); + Path location = part.getLocation(tablePath, partSpec); Partition partition = MetaStoreServerUtils.createMetaPartitionObject(table, partSpec, location); partition.setWriteId(table.getWriteId()); partsToAdd.add(partition); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index 92d10cd0e1..f9d7c13e1b 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -1449,12 +1449,16 @@ public static String getPartitionName(Path tablePath, Path partitionPath, Set