diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 3437fc8..14f7374 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -497,12 +497,16 @@ public static Directory getAcidState(Path directory, // If we have a base, the original files are obsolete. if (bestBase.status != null) { + // Add original files to obsolete list if any for (HdfsFileStatusWithId fswid : original) { obsolete.add(fswid.getFileStatus()); } + // Add original direcotries to obsolete list if any + obsolete.addAll(originalDirectories); // remove the entries so we don't get confused later and think we should // use them. original.clear(); + originalDirectories.clear(); } else { // Okay, we're going to need these originals. Recurse through them and figure out what we // really need. diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 890012a..b784585 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -348,6 +348,18 @@ public void testNonAcidToAcidConversionAndMajorCompaction() throws Exception { Assert.assertEquals(resultCount, Integer.parseInt(rs.get(0))); // 5. Let Cleaner delete obsolete files/dirs + // Note, here we create a fake directory along with fake files as original directories/files + String fakeFile0 = TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase() + + "/subdir/000000_0"; + String fakeFile1 = TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase() + + "/subdir/000000_1"; + fs.create(new Path(fakeFile0)); + fs.create(new Path(fakeFile1)); + status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + + (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER); + // Before Cleaner, there should be 5 items: + // 2 original files, 1 original directory, 1 base directory and 1 delta directory + Assert.assertEquals(5, status.length); Cleaner c = new Cleaner(); c.setThreadId((int) c.getId()); c.setHiveConf(hiveConf);