diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java index d9e206c..6ade76d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java @@ -175,19 +175,19 @@ public void testRecyclePartTable() throws Exception { // verify cm.recycle(db, table, part) api moves file to cmroot dir int ret = cm.recycle(part1Path, RecycleType.MOVE, false); Assert.assertEquals(ret, 1); - Path cmPart1Path = ReplChangeManager.getCMPath(hiveConf, path1Chksum); + Path cmPart1Path = ReplChangeManager.getCMPath(hiveConf, part1Path.getName(), path1Chksum); assertTrue(cmPart1Path.getFileSystem(hiveConf).exists(cmPart1Path)); // Verify dropPartition recycle part files client.dropPartition(dbName, tblName, Arrays.asList("20160102")); assertFalse(part2Path.getFileSystem(hiveConf).exists(part2Path)); - Path cmPart2Path = ReplChangeManager.getCMPath(hiveConf, path2Chksum); + Path cmPart2Path = ReplChangeManager.getCMPath(hiveConf, part2Path.getName(), path2Chksum); assertTrue(cmPart2Path.getFileSystem(hiveConf).exists(cmPart2Path)); // Verify dropTable recycle partition files client.dropTable(dbName, tblName); assertFalse(part3Path.getFileSystem(hiveConf).exists(part3Path)); - Path cmPart3Path = ReplChangeManager.getCMPath(hiveConf, path3Chksum); + Path cmPart3Path = ReplChangeManager.getCMPath(hiveConf, part3Path.getName(), path3Chksum); assertTrue(cmPart3Path.getFileSystem(hiveConf).exists(cmPart3Path)); client.dropDatabase(dbName, true, true); @@ -246,17 +246,17 @@ public void testRecycleNonPartTable() throws Exception { cm.recycle(filePath1, RecycleType.MOVE, false); assertFalse(filePath1.getFileSystem(hiveConf).exists(filePath1)); - Path cmPath1 = ReplChangeManager.getCMPath(hiveConf, fileChksum1); + Path cmPath1 = ReplChangeManager.getCMPath(hiveConf, filePath1.getName(), fileChksum1); assertTrue(cmPath1.getFileSystem(hiveConf).exists(cmPath1)); // Verify dropTable recycle table files client.dropTable(dbName, tblName); - Path cmPath2 = ReplChangeManager.getCMPath(hiveConf, fileChksum2); + Path cmPath2 = ReplChangeManager.getCMPath(hiveConf, filePath2.getName(), fileChksum2); assertFalse(filePath2.getFileSystem(hiveConf).exists(filePath2)); assertTrue(cmPath2.getFileSystem(hiveConf).exists(cmPath2)); - Path cmPath3 = ReplChangeManager.getCMPath(hiveConf, fileChksum3); + Path cmPath3 = ReplChangeManager.getCMPath(hiveConf, filePath3.getName(), fileChksum3); assertFalse(filePath3.getFileSystem(hiveConf).exists(filePath3)); assertTrue(cmPath3.getFileSystem(hiveConf).exists(cmPath3)); @@ -298,17 +298,17 @@ public void testClearer() throws Exception { ReplChangeManager.getInstance(hiveConf).recycle(dirTbl2, RecycleType.MOVE, false); ReplChangeManager.getInstance(hiveConf).recycle(dirTbl3, RecycleType.MOVE, true); - assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum11))); - assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum12))); - assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum21))); - assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum22))); - assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31))); - assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum32))); + assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11))); + assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part12.getName(), fileChksum12))); + assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21))); + assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part22.getName(), fileChksum22))); + assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31))); + assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32))); - fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum11), now - 86400*1000*2, now - 86400*1000*2); - fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum21), now - 86400*1000*2, now - 86400*1000*2); - fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum31), now - 86400*1000*2, now - 86400*1000*2); - fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum32), now - 86400*1000*2, now - 86400*1000*2); + fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11), now - 86400*1000*2, now - 86400*1000*2); + fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21), now - 86400*1000*2, now - 86400*1000*2); + fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31), now - 86400*1000*2, now - 86400*1000*2); + fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32), now - 86400*1000*2, now - 86400*1000*2); ReplChangeManager.scheduleCMClearer(hiveConf); @@ -321,12 +321,12 @@ public void testClearer() throws Exception { if (end - start > 5000) { Assert.fail("timeout, cmroot has not been cleared"); } - if (!fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum11)) && - fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum12)) && - !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum21)) && - fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum22)) && - !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31)) && - !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31))) { + if (!fs.exists(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11)) && + fs.exists(ReplChangeManager.getCMPath(hiveConf, part12.getName(), fileChksum12)) && + !fs.exists(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21)) && + fs.exists(ReplChangeManager.getCMPath(hiveConf, part22.getName(), fileChksum22)) && + !fs.exists(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31)) && + !fs.exists(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32))) { cleared = true; } } while (!cleared); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java index 88d6a7a..6cb5fa8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java @@ -132,7 +132,7 @@ int recycle(Path path, RecycleType type, boolean ifPurge) throws MetaException { } } else { String fileCheckSum = checksumFor(path, fs); - Path cmPath = getCMPath(hiveConf, fileCheckSum); + Path cmPath = getCMPath(hiveConf, path.getName(), fileCheckSum); // set timestamp before moving to cmroot, so we can // avoid race condition CM remove the file before setting @@ -237,11 +237,12 @@ static public void setCmRoot(Path cmRoot) { * to a deterministic location of cmroot. So user can retrieve the file back * with the original location plus checksum. * @param conf + * @param name original filename * @param checkSum checksum of the file, can be retrieved by {@link #checksumFor(Path, FileSystem)} * @return Path */ - static Path getCMPath(Configuration conf, String checkSum) throws IOException, MetaException { - String newFileName = checkSum; + static Path getCMPath(Configuration conf, String name, String checkSum) throws IOException, MetaException { + String newFileName = name + "_" + checkSum; int maxLength = conf.getInt(DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY, DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT); @@ -269,14 +270,14 @@ static public FileStatus getFileStatus(Path src, String checksumString, } if (!srcFs.exists(src)) { - return srcFs.getFileStatus(getCMPath(hiveConf, checksumString)); + return srcFs.getFileStatus(getCMPath(hiveConf, src.getName(), checksumString)); } String currentChecksumString = checksumFor(src, srcFs); if (currentChecksumString == null || checksumString.equals(currentChecksumString)) { return srcFs.getFileStatus(src); } else { - return srcFs.getFileStatus(getCMPath(hiveConf, checksumString)); + return srcFs.getFileStatus(getCMPath(hiveConf, src.getName(), checksumString)); } } catch (IOException e) { throw new MetaException(StringUtils.stringifyException(e));