diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java index e288571..665b5e2 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java @@ -86,6 +86,10 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest { public List listFiles() { return Collections.unmodifiableList(files); } + + public int getFileCount () { + return files.size(); + } } /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java index 33f5e71..2226466 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -187,6 +188,25 @@ public class PartitionedMobCompactor extends MobCompactor { selectedFileCount++; } } + + /* + * If it is not a major mob compaction with del files, and the file number in Partition is 1, + * remove the partition from filesToCompact list to avoid re-compacting files which has been + * compacted with del files. + */ + if (!allFiles && (allDelFiles.size() > 0)) { + Iterator> it = + filesToCompact.entrySet().iterator(); + + while(it.hasNext()) { + Map.Entry entry = it.next(); + if (entry.getValue().getFileCount() <= 1) { + it.remove(); + --selectedFileCount; + } + } + } + PartitionedMobCompactionRequest request = new PartitionedMobCompactionRequest( filesToCompact.values(), allDelFiles); if (candidates.size() == (allDelFiles.size() + selectedFileCount + irrelevantFileCount)) { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java index 7da8544..c112289 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java @@ -106,11 +106,23 @@ public class TestPartitionedMobCompactor { @Test public void testCompactionSelectWithAllFiles() throws Exception { String tableName = "testCompactionSelectWithAllFiles"; + // If there is only 1 file, it will not be compacted with _del files, so + // It wont be CompactionType.ALL_FILES in this case, do not create with _del files. testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD, - CompactionType.ALL_FILES, false); + CompactionType.ALL_FILES, false, false); } @Test + public void testCompactionSelectToAvoidCompactOneFileWithDelete() throws Exception { + String tableName = "testCompactionSelectToAvoidCompactOneFileWithDelete"; + // If there is only 1 file, it will not be compacted with _del files, so + // It wont be CompactionType.ALL_FILES in this case, and expected compact file count will be 0. + testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD, + CompactionType.PART_FILES, false); + } + + + @Test public void testCompactionSelectWithPartFiles() throws Exception { String tableName = "testCompactionSelectWithPartFiles"; testCompactionAtMergeSize(tableName, 4000, CompactionType.PART_FILES, false); @@ -125,20 +137,36 @@ public class TestPartitionedMobCompactor { private void testCompactionAtMergeSize(final String tableName, final long mergeSize, final CompactionType type, final boolean isForceAllFiles) throws Exception { + testCompactionAtMergeSize(tableName, mergeSize, type, isForceAllFiles, true); + } + + private void testCompactionAtMergeSize(final String tableName, + final long mergeSize, final CompactionType type, final boolean isForceAllFiles, + final boolean createDelFiles) + throws Exception { resetConf(); init(tableName); int count = 10; // create 10 mob files. createStoreFiles(basePath, family, qf, count, Type.Put); - // create 10 del files - createStoreFiles(basePath, family, qf, count, Type.Delete); + + if (createDelFiles) { + // create 10 del files + createStoreFiles(basePath, family, qf, count, Type.Delete); + } + listFiles(); List expectedStartKeys = new ArrayList<>(); for(FileStatus file : mobFiles) { if(file.getLen() < mergeSize) { String fileName = file.getPath().getName(); String startKey = fileName.substring(0, 32); - expectedStartKeys.add(startKey); + + // If it is not an major mob compaction and del files are there, + // these mob files wont be compacted. + if (isForceAllFiles || !createDelFiles) { + expectedStartKeys.add(startKey); + } } } // set the mob compaction mergeable threshold @@ -150,7 +178,7 @@ public class TestPartitionedMobCompactor { public void testCompactDelFilesWithDefaultBatchSize() throws Exception { String tableName = "testCompactDelFilesWithDefaultBatchSize"; testCompactDelFilesAtBatchSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE, - MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); + MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); } @Test