diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java index 520eb1b5c0..198fb81076 100644 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java @@ -116,7 +116,7 @@ public void tearDown() throws Exception { @Test public void testTableCheck() throws HiveException, IOException, TException, MetastoreException { CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, null, null, result); + checker.checkMetastore(catName, dbName, null, null, null, result); // we haven't added anything so should return an all ok assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); @@ -125,7 +125,7 @@ public void testTableCheck() throws HiveException, IOException, TException, Meta // check table only, should not exist in ms result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(1, result.getTablesNotInMs().size()); assertEquals(tableName, result.getTablesNotInMs().iterator().next()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); @@ -148,7 +148,7 @@ public void testTableCheck() throws HiveException, IOException, TException, Meta // now we've got a table, check that it works // first check all (1) tables result = new CheckResult(); - checker.checkMetastore(catName, dbName, null, null, result); + checker.checkMetastore(catName, dbName, null, null, null, result); assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.emptySet(), result.getPartitionsNotOnFs()); @@ -156,7 +156,7 @@ public void testTableCheck() throws HiveException, IOException, TException, Meta // then let's check the one we know about result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.emptySet(), result.getPartitionsNotOnFs()); @@ -168,7 +168,7 @@ public void testTableCheck() throws HiveException, IOException, TException, Meta // now this shouldn't find the path on the fs result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(1, result.getTablesNotOnFs().size()); assertEquals(tableName, result.getTablesNotOnFs().iterator().next()); @@ -184,7 +184,7 @@ public void testTableCheck() throws HiveException, IOException, TException, Meta // find the extra table result = new CheckResult(); - checker.checkMetastore(catName, dbName, null, null, result); + checker.checkMetastore(catName, dbName, null, null, null, result); assertEquals(1, result.getTablesNotInMs().size()); assertEquals(fakeTable.getName(), Lists.newArrayList(result.getTablesNotInMs()).get(0)); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); @@ -198,7 +198,7 @@ public void testTableCheck() throws HiveException, IOException, TException, Meta // should return all ok result = new CheckResult(); - checker.checkMetastore(catName, dbName, null, null, result); + checker.checkMetastore(catName, dbName, null, null, null, result); assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.emptySet(), result.getPartitionsNotOnFs()); @@ -222,7 +222,7 @@ public void testAdditionalPartitionDirs() fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections. emptySet(), result.getTablesNotInMs()); assertEquals(Collections. emptySet(), result.getTablesNotOnFs()); assertEquals(Collections. emptySet(), result.getPartitionsNotOnFs()); @@ -242,7 +242,7 @@ public void testInvalidPartitionKeyName() "fakedate=2009-01-01/fakecity=sanjose"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); - checker.checkMetastore(catName, dbName, tableName, null, new CheckResult()); + checker.checkMetastore(catName, dbName, tableName, null, null, new CheckResult()); } /* @@ -265,7 +265,7 @@ public void testSkipInvalidPartitionKeyName() fs.deleteOnExit(fakePart); createPartitionsDirectoriesOnFS(table, 2); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections. emptySet(), result.getTablesNotInMs()); assertEquals(Collections. emptySet(), result.getTablesNotOnFs()); assertEquals(Collections. emptySet(), result.getPartitionsNotOnFs()); @@ -301,7 +301,7 @@ public void testPartitionsCheck() throws HiveException, Table table = createTestTable(); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); // all is well assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); @@ -320,7 +320,7 @@ public void testPartitionsCheck() throws HiveException, fs.delete(partToRemovePath, true); result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); // missing one partition on fs assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); @@ -335,7 +335,7 @@ public void testPartitionsCheck() throws HiveException, partsCopy.add(partitions.get(1).getSpec()); // check only the partition that exists, all should be well result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, partsCopy, result); + checker.checkMetastore(catName, dbName, tableName, partsCopy, null, result); assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.emptySet(), result.getPartitionsNotOnFs()); @@ -347,7 +347,7 @@ public void testPartitionsCheck() throws HiveException, hive.dropTable(dbName, tableName, true, true); hive.createTable(table); result = new CheckResult(); - checker.checkMetastore(catName, dbName, null, null, result); + checker.checkMetastore(catName, dbName, null, null, null, result); assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.emptySet(), result.getPartitionsNotOnFs()); @@ -398,7 +398,7 @@ public void testPartitionsNotInMs() throws HiveException, AlreadyExistsException // add 10 partitions on the filesystem createPartitionsDirectoriesOnFS(testTable, 10); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections.emptySet(), result.getTablesNotInMs()); assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.emptySet(), result.getPartitionsNotOnFs()); @@ -417,7 +417,7 @@ public void testSingleThreadedCheckMetastore() // add 10 partitions on the filesystem createPartitionsDirectoriesOnFS(testTable, 10); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections. emptySet(), result.getTablesNotInMs()); assertEquals(Collections. emptySet(), result.getTablesNotOnFs()); assertEquals(Collections. emptySet(), result.getPartitionsNotOnFs()); @@ -442,7 +442,7 @@ public void testSingleThreadedDeeplyNestedTables() // add 10 partitions on the filesystem createPartitionsDirectoriesOnFS(testTable, 10); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections. emptySet(), result.getTablesNotInMs()); assertEquals(Collections. emptySet(), result.getTablesNotOnFs()); assertEquals(Collections. emptySet(), result.getPartitionsNotOnFs()); @@ -466,7 +466,7 @@ public void testDeeplyNestedPartitionedTables() // add 10 partitions on the filesystem createPartitionsDirectoriesOnFS(testTable, 10); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections. emptySet(), result.getTablesNotInMs()); assertEquals(Collections. emptySet(), result.getTablesNotOnFs()); assertEquals(Collections. emptySet(), result.getPartitionsNotOnFs()); @@ -494,7 +494,7 @@ public void testErrorForMissingPartitionColumn() throws AlreadyExistsException, CheckResult result = new CheckResult(); Exception exception = null; try { - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); } catch (Exception e) { exception = e; } @@ -503,7 +503,7 @@ public void testErrorForMissingPartitionColumn() throws AlreadyExistsException, result = new CheckResult(); exception = null; try { - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); } catch (Exception e) { exception = e; } @@ -525,7 +525,7 @@ public void testInvalidOrderForPartitionKeysOnFS() // add 10 partitions on the filesystem createInvalidPartitionDirsOnFS(testTable, 10); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); } /* @@ -543,7 +543,7 @@ public void testSkipInvalidOrderForPartitionKeysOnFS() // add 10 partitions on the filesystem createPartitionsDirectoriesOnFS(testTable, 2); CheckResult result = new CheckResult(); - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); assertEquals(Collections. emptySet(), result.getTablesNotInMs()); assertEquals(Collections. emptySet(), result.getTablesNotOnFs()); assertEquals(Collections. emptySet(), result.getPartitionsNotOnFs()); @@ -572,7 +572,7 @@ public void testErrorForMissingPartitionsSingleThreaded() CheckResult result = new CheckResult(); Exception exception = null; try { - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); } catch (Exception e) { exception = e; } @@ -581,7 +581,7 @@ public void testErrorForMissingPartitionsSingleThreaded() result = new CheckResult(); exception = null; try { - checker.checkMetastore(catName, dbName, tableName, null, result); + checker.checkMetastore(catName, dbName, tableName, null, null, result); } catch (Exception e) { exception = e; } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java index 6f4400a8ef..9b454cc199 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java @@ -111,6 +111,7 @@ public IMetaStoreClient getMsc() { * @param partitions * List of partition name value pairs, if null or empty check all * partitions + * @param table * @param result * Fill this with the results of the check * @throws MetastoreException @@ -119,7 +120,7 @@ public IMetaStoreClient getMsc() { * Most likely filesystem related */ public void checkMetastore(String catName, String dbName, String tableName, - List> partitions, CheckResult result) + List> partitions, Table table, CheckResult result) throws MetastoreException, IOException { if (dbName == null || "".equalsIgnoreCase(dbName)) { @@ -131,16 +132,16 @@ public void checkMetastore(String catName, String dbName, String tableName, // no table specified, check all tables and all partitions. List tables = getMsc().getTables(catName, dbName, ".*"); for (String currentTableName : tables) { - checkTable(catName, dbName, currentTableName, null, result); + checkTable(catName, dbName, currentTableName, null, null, result); } findUnknownTables(catName, dbName, tables, result); } else if (partitions == null || partitions.isEmpty()) { // only one table, let's check all partitions - checkTable(catName, dbName, tableName, null, result); + checkTable(catName, dbName, tableName, null, table, result); } else { // check the specified partitions - checkTable(catName, dbName, tableName, partitions, result); + checkTable(catName, dbName, tableName, partitions, table, result); } LOG.info("Number of partitionsNotInMs=" + result.getPartitionsNotInMs() + ", partitionsNotOnFs=" + result.getPartitionsNotOnFs() @@ -215,6 +216,7 @@ void findUnknownTables(String catName, String dbName, List tables, Check * Name of the table * @param partitions * Partitions to check, if null or empty get all the partitions. + * @param table * @param result * Result object * @throws MetastoreException @@ -225,16 +227,16 @@ void findUnknownTables(String catName, String dbName, List tables, Check * Failed to get required information from the metastore. */ void checkTable(String catName, String dbName, String tableName, - List> partitions, CheckResult result) + List> partitions, Table table, CheckResult result) throws MetaException, IOException, MetastoreException { - Table table; - - try { - table = getMsc().getTable(catName, dbName, tableName); - } catch (TException e) { - result.getTablesNotInMs().add(tableName); - return; + if (table == null) { + try { + table = getMsc().getTable(catName, dbName, tableName); + } catch (TException e) { + result.getTablesNotInMs().add(tableName); + return; + } } PartitionIterable parts; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java index f4e109d1b0..974b175742 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java @@ -118,7 +118,7 @@ public int repair(MsckInfo msckInfo) { // And partitions that are not present in filesystem and metadata exists in metastore - // accessed through getPartitionNotOnFS checker.checkMetastore(msckInfo.getCatalogName(), msckInfo.getDbName(), msckInfo.getTableName(), - msckInfo.getPartSpecs(), result); + msckInfo.getPartSpecs(), table, result); Set partsNotInMs = result.getPartitionsNotInMs(); Set partsNotInFs = result.getPartitionsNotOnFs(); Set expiredPartitions = result.getExpiredPartitions();