diff --git ql/src/test/queries/clientpositive/msck_repair_4.q ql/src/test/queries/clientpositive/msck_repair_4.q new file mode 100644 index 0000000000..8cee91c440 --- /dev/null +++ ql/src/test/queries/clientpositive/msck_repair_4.q @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS repairtable_n4; + +CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC; + +MSCK TABLE repairtable_n4; +show partitions repairtable_n4; + +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/Year=2020/Month=3/Day=1; +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/Year=2020/Month=3/Day=2; + +MSCK TABLE repairtable_n4; +show partitions repairtable_n4; + +DROP TABLE default.repairtable_n3; diff --git ql/src/test/results/clientpositive/msck_repair_4.q.out ql/src/test/results/clientpositive/msck_repair_4.q.out new file mode 100644 index 0000000000..e7681543a6 --- /dev/null +++ ql/src/test/results/clientpositive/msck_repair_4.q.out @@ -0,0 +1,41 @@ +PREHOOK: query: DROP TABLE IF EXISTS repairtable_n4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@repairtable_n4 +POSTHOOK: query: CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repairtable_n4 +PREHOOK: query: MSCK TABLE repairtable_n4 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n4 +POSTHOOK: query: MSCK TABLE repairtable_n4 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n4 +PREHOOK: query: show partitions repairtable_n4 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n4 +POSTHOOK: query: show partitions repairtable_n4 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n4 +PREHOOK: query: MSCK TABLE repairtable_n4 +PREHOOK: type: MSCK +PREHOOK: Output: default@repairtable_n4 +POSTHOOK: query: MSCK TABLE repairtable_n4 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable_n4 +Partitions not in metastore: repairtable_n4:year=2020/month=3/day=1 repairtable_n4:year=2020/month=3/day=2 +PREHOOK: query: show partitions repairtable_n4 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@repairtable_n4 +POSTHOOK: query: show partitions repairtable_n4 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@repairtable_n4 +PREHOOK: query: DROP TABLE default.repairtable_n3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE default.repairtable_n3 +POSTHOOK: type: DROPTABLE diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java index 5287f47e21..f2389f52c6 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.MetaException; + +import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -108,6 +112,8 @@ public void setExpiredPartitions( public static class PartitionResult implements Comparable { private String partitionName; private String tableName; + private Path path; + private boolean pathSet; /** * @return name of partition @@ -139,6 +145,22 @@ public void setTableName(String tableName) { this.tableName = tableName; } + public void setPathSet(boolean pathSet) { + this.pathSet = pathSet; + } + + public void setPath(Path path) { + this.path = path; + } + + public Path getLocation(Path tablePath, Map partSpec) throws MetaException { + if (!this.pathSet) { + return new Path(tablePath, Warehouse.makePartPath(partSpec)); + } + + return this.path; + } + @Override public String toString() { return tableName + ":" + partitionName; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java index 6f4400a8ef..26e20eb687 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java @@ -401,6 +401,11 @@ void findUnknownPartitions(Table table, Set partPaths, CheckResult.PartitionResult pr = new CheckResult.PartitionResult(); pr.setPartitionName(partitionName); pr.setTableName(table.getTableName()); + // Also set the correct partition path here as creating path from Warehouse.makePartPath will always return + // lowercase keys/path. Even if we add the new partition with lowerkeys, get queries on such partition + // will not return any results. + pr.setPath(partPath); + pr.setPathSet(true); result.getPartitionsNotInMs().add(pr); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java index f4e109d1b0..ab0c649f90 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java @@ -373,7 +373,7 @@ public Void execute(int size) throws MetastoreException { continue; } Map partSpec = Warehouse.makeSpecFromName(part.getPartitionName()); - Path location = new Path(tablePath, Warehouse.makePartPath(partSpec)); + Path location = part.getLocation(tablePath, partSpec); Partition partition = MetaStoreServerUtils.createMetaPartitionObject(table, partSpec, location); partition.setWriteId(table.getWriteId()); partsToAdd.add(partition); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index 7c4e129738..6251a2e134 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -1448,12 +1448,16 @@ public static String getPartitionName(Path tablePath, Path partitionPath, Set