diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 986649a..6da7655 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -68,8 +68,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.mob.compactions.MobCompactor; -import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -81,10 +79,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -92,17 +88,16 @@ @Category(LargeTests.class) public class TestMobCompactor { private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private Configuration conf = null; - private String tableNameAsString; + private static Configuration conf = null; private TableName tableName; private static Connection conn; private BufferedMutator bufMut; - private Table hTable; - private Admin admin; + private Table table; + private static Admin admin; private HTableDescriptor desc; private HColumnDescriptor hcd1; private HColumnDescriptor hcd2; - private FileSystem fs; + private static FileSystem fs; private static final String family1 = "family1"; private static final String family2 = "family2"; private static final String qf1 = "qualifier1"; @@ -127,6 +122,9 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(1); pool = createThreadPool(TEST_UTIL.getConfiguration()); conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + admin = TEST_UTIL.getHBaseAdmin(); } @AfterClass @@ -136,249 +134,128 @@ public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - @Before - public void setUp() throws Exception { - fs = TEST_UTIL.getTestFileSystem(); - conf = TEST_UTIL.getConfiguration(); - long tid = System.currentTimeMillis(); - tableNameAsString = "testMob" + tid; + public void setUp(String tableNameAsString) throws IOException { tableName = TableName.valueOf(tableNameAsString); hcd1 = new HColumnDescriptor(family1); hcd1.setMobEnabled(true); hcd1.setMobThreshold(5); - hcd1.setMaxVersions(4); hcd2 = new HColumnDescriptor(family2); hcd2.setMobEnabled(true); hcd2.setMobThreshold(5); - hcd2.setMaxVersions(4); desc = new HTableDescriptor(tableName); desc.addFamily(hcd1); desc.addFamily(hcd2); - admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(desc, getSplitKeys()); - hTable = conn.getTable(tableName); + table = conn.getTable(tableName); bufMut = conn.getBufferedMutator(tableName); } - @After - public void tearDown() throws Exception { - admin.disableTable(tableName); - admin.deleteTable(tableName); - admin.close(); - hTable.close(); - fs.delete(TEST_UTIL.getDataTestDir(), true); - } - @Test(timeout = 300000) - public void testCompactionWithoutDelFiles() throws Exception { + public void testMinorCompaction() throws Exception { resetConf(); + int mergeSize = 5000; + // change the mob compaction merge size + conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); + // create a table with namespace NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build(); - String tableNameAsString = "ns:testCompactionWithoutDelFiles"; + String tableNameAsString = "ns:testMinorCompaction"; admin.createNamespace(namespaceDescriptor); - Configuration conf = TEST_UTIL.getConfiguration(); - SecureRandom rng = new SecureRandom(); - byte[] keyBytes = new byte[AES.KEY_LENGTH]; - rng.nextBytes(keyBytes); - String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); - Key cfKey = new SecretKeySpec(keyBytes, algorithm); - byte[] encryptionKey = EncryptionUtil.wrapKey(conf, - conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey); - TableName tableName = TableName.valueOf(tableNameAsString); - HTableDescriptor desc = new HTableDescriptor(tableName); - HColumnDescriptor hcd = new HColumnDescriptor(family1); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0); - hcd.setMaxVersions(4); - hcd.setEncryptionType(algorithm); - hcd.setEncryptionKey(encryptionKey); - HColumnDescriptor hcd2 = new HColumnDescriptor(family2); - hcd2.setMobEnabled(true); - hcd2.setMobThreshold(0); - hcd2.setMaxVersions(4); - desc.addFamily(hcd); - desc.addFamily(hcd2); - admin.createTable(desc, getSplitKeys()); - Table table = conn.getTable(tableName); - BufferedMutator bufMut = conn.getBufferedMutator(tableName); + setUp(tableNameAsString); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); int rowNumPerRegion = count * rowNumPerFile; - assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion, + assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, countMobRows(table)); - assertEquals("Before compaction: mob file count", regionNum * count, + assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, + countMobCells(table)); + assertEquals("Before deleting: mob file count", regionNum * count, countFiles(tableName, true, family1)); - assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1)); - MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd, pool); - compactor.compact(); + int largeFilesCount = countLargeFiles(mergeSize, tableName, family1); + createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); - assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion, + assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), countMobRows(table)); - assertEquals("After compaction: mob file count", regionNum, + assertEquals("Before compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); + assertEquals("Before compaction: family1 mob file count", regionNum * count, countFiles(tableName, true, family1)); - assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1)); - Assert.assertTrue(verifyEncryption(tableName, family1)); - table.close(); - admin.disableTable(tableName); - admin.deleteTable(tableName); - admin.deleteNamespace("ns"); - } - - @Test(timeout = 300000) - public void testCompactionWithDelFiles() throws Exception { - resetConf(); - int batchSize = 2; - conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize); - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before deleting: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("Before compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); - assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - - // do the mob file compaction - MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); - compactor.compact(); - - assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize), - countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("After compaction: family1 del file count", 0, countFiles(tableName, false, family1)); - assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - assertRefFileNameEqual(family1); - } - - @Test(timeout = 300000) - public void testMinorCompactionWithDelFiles() throws Exception { - resetConf(); - int mergeSize = 5000; - // change the mob compaction merge size - conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); - - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: mob file count", regionNum * count, - countFiles(tableName, true, family1)); - - int largeFilesCount = countLargeFiles(mergeSize, family1); - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); // do the mob file compaction MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); compactor.compact(); - assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); + assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), + countMobRows(table)); + assertEquals("After compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); // After the compaction, the files smaller than the mob compaction merge size // is merge to one file assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum, - countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + countFiles(tableName, true, family1)); + assertEquals("After compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); } @Test(timeout = 300000) public void testCompactionWithHFileLink() throws IOException, InterruptedException { resetConf(); + String tableNameAsString = "testCompactionWithHFileLink"; + setUp(tableNameAsString); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; + int rowNumPerRegion = count * rowNumPerFile; long tid = System.currentTimeMillis(); byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid); // take a snapshot admin.snapshot(snapshotName1, tableName); - createDelFile(); + createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), + countMobRows(table)); + assertEquals("Before compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); + assertEquals("Before compaction: family1 mob file count", regionNum * count, + countFiles(tableName, true, family1)); + assertEquals("Before compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); // do the mob compaction MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); compactor.compact(); - assertEquals("After first compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After first compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); + assertEquals("After first compaction: mob rows count", regionNum + * (rowNumPerRegion - delRowNum), countMobRows(table)); + assertEquals("After first compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After first compaction: family1 mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After first compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + countFiles(tableName, true, family1)); + assertEquals("After first compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After first compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After first compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1)); assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2)); @@ -387,33 +264,32 @@ public void testCompactionWithHFileLink() throws IOException, InterruptedExcepti admin.restoreSnapshot(snapshotName1); admin.enableTable(tableName); - assertEquals("After restoring snapshot: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("After restoring snapshot: mob cells count", - regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable)); - assertEquals("After restoring snapshot: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("After restoring snapshot: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion, + countMobRows(table)); + assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow + * rowNumPerRegion, countMobCells(table)); + assertEquals("After restoring snapshot: family1 mob file count", regionNum * count, + countFiles(tableName, true, family1)); + assertEquals("After restoring snapshot: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After restoring snapshot: family1 del file count", 0, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("After restoring snapshot: family2 del file count", 0, - countFiles(tableName, false, family2)); - assertEquals("After restoring snapshot: family1 hfilelink count", regionNum*count, - countHFileLinks(family1)); - assertEquals("After restoring snapshot: family2 hfilelink count", 0, - countHFileLinks(family2)); + countFiles(tableName, false, family2)); + assertEquals("After restoring snapshot: family1 hfilelink count", regionNum * count, + countHFileLinks(family1)); + assertEquals("After restoring snapshot: family2 hfilelink count", 0, countHFileLinks(family2)); compactor.compact(); - assertEquals("After second compaction: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("After second compaction: mob cells count", - regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable)); + assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion, + countMobRows(table)); + assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow + * rowNumPerRegion, countMobCells(table)); assertEquals("After second compaction: family1 mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After second compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + countFiles(tableName, true, family1)); + assertEquals("After second compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After second compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After second compaction: family2 del file count", 0, @@ -424,114 +300,85 @@ public void testCompactionWithHFileLink() throws IOException, InterruptedExcepti } @Test(timeout = 300000) - public void testCompactionFromAdmin() throws Exception { + public void testMajorCompactionFromAdmin() throws Exception { resetConf(); int mergeSize = 5000; // change the mob compaction merge size conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); + String tableNameAsString = "testMajorCompactionFromAdmin"; + SecureRandom rng = new SecureRandom(); + byte[] keyBytes = new byte[AES.KEY_LENGTH]; + rng.nextBytes(keyBytes); + String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Key cfKey = new SecretKeySpec(keyBytes, algorithm); + byte[] encryptionKey = EncryptionUtil.wrapKey(conf, + conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey); + TableName tableName = TableName.valueOf(tableNameAsString); + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor hcd1 = new HColumnDescriptor(family1); + hcd1.setMobEnabled(true); + hcd1.setMobThreshold(0); + hcd1.setEncryptionType(algorithm); + hcd1.setEncryptionKey(encryptionKey); + HColumnDescriptor hcd2 = new HColumnDescriptor(family2); + hcd2.setMobEnabled(true); + hcd2.setMobThreshold(0); + desc.addFamily(hcd1); + desc.addFamily(hcd2); + admin.createTable(desc, getSplitKeys()); + Table table = conn.getTable(tableName); + BufferedMutator bufMut = conn.getBufferedMutator(tableName); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before deleting: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); - assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + int rowNumPerRegion = count * rowNumPerFile; - int largeFilesCount = countLargeFiles(5000, family1); - // do the mob compaction - admin.compact(tableName, hcd1.getName(), Admin.CompactType.MOB); + assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, + countMobRows(table)); + assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, + countMobCells(table)); + assertEquals("Before deleting: mob file count", regionNum * count, + countFiles(tableName, true, family1)); - waitUntilMobCompactionFinished(tableName); - assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", regionNum - * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(hTable)); - assertEquals("After compaction: family1 mob file count", regionNum + largeFilesCount, + createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); + + assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), + countMobRows(table)); + assertEquals("Before compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); + assertEquals("Before compaction: family1 mob file count", regionNum * count, countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum * count, + assertEquals("Before compaction: family2 mob file count", regionNum * count, countFiles(tableName, true, family2)); - assertEquals("After compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); - assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - assertRefFileNameEqual(family1); - } - - @Test(timeout = 300000) - public void testMajorCompactionFromAdmin() throws Exception { - resetConf(); - int mergeSize = 5000; - // change the mob compaction merge size - conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: mob file count", regionNum*count, - countFiles(tableName, true, family1)); - - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); // do the major mob compaction, it will force all files to compaction admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB); waitUntilMobCompactionFinished(tableName); - assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); + assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), + countMobRows(table)); + assertEquals("After compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After compaction: family1 mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + countFiles(tableName, true, family1)); + assertEquals("After compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After compaction: family1 del file count", 0, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); + Assert.assertTrue(verifyEncryption(tableName, family1)); + table.close(); } @Test(timeout = 300000) public void testScannerOnBulkLoadRefHFiles() throws Exception { resetConf(); + setUp("testScannerOnBulkLoadRefHFiles"); long ts = EnvironmentEdgeManager.currentTime(); byte[] key0 = Bytes.toBytes("k0"); byte[] key1 = Bytes.toBytes("k1"); @@ -548,7 +395,7 @@ public void testScannerOnBulkLoadRefHFiles() throws Exception { loadData(admin, bufMut, tableName, new Put[] { put0, put1 }); // read the latest cell of key0. Get get = new Get(key0); - Result result = hTable.get(get); + Result result = table.get(get); Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob value of k0", newValue0, Bytes.toString(CellUtil.cloneValue(cell))); @@ -556,18 +403,18 @@ public void testScannerOnBulkLoadRefHFiles() throws Exception { waitUntilMobCompactionFinished(tableName); // read the latest cell of key0, the cell seqId in bulk loaded file is not reset in the // scanner. The cell that has "new" value is still visible. - result = hTable.get(get); + result = table.get(get); cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); assertEquals("After compaction: mob value of k0", newValue0, Bytes.toString(CellUtil.cloneValue(cell))); // read the ref cell, not read further to the mob cell. get = new Get(key1); get.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(true)); - result = hTable.get(get); + result = table.get(get); cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); // the ref name is the new file - Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), - tableName), hcd1.getNameAsString()); + Path mobFamilyPath = new Path( + MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), hcd1.getNameAsString()); List paths = new ArrayList(); if (fs.exists(mobFamilyPath)) { FileStatus[] files = fs.listStatus(mobFamilyPath); @@ -585,6 +432,7 @@ public void testScannerOnBulkLoadRefHFiles() throws Exception { @Test(timeout = 300000) public void testScannerAfterCompactions() throws Exception { resetConf(); + setUp("testScannerAfterCompactions"); long ts = EnvironmentEdgeManager.currentTime(); byte[] key0 = Bytes.toBytes("k0"); byte[] key1 = Bytes.toBytes("k1"); @@ -605,7 +453,7 @@ public void testScannerAfterCompactions() throws Exception { waitUntilMobCompactionFinished(tableName); // read the latest cell of key1. Get get = new Get(key1); - Result result = hTable.get(get); + Result result = table.get(get); Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); assertEquals("After compaction: mob value", "new", Bytes.toString(CellUtil.cloneValue(cell))); } @@ -748,19 +596,19 @@ private int countHFileLinks(String familyName) throws IOException { /** * Gets the number of files. * @param size the size of the file + * @param tableName the current table name * @param familyName the family name * @return the number of files large than the size */ - private int countLargeFiles(int size, String familyName) throws IOException { - Path mobDirPath = MobUtils.getMobFamilyPath( - MobUtils.getMobRegionPath(conf, tableName), familyName); + private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException { + Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName), + familyName); int count = 0; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); for (FileStatus file : files) { // ignore the del files in the mob path - if ((!StoreFileInfo.isDelFile(file.getPath())) - && (file.getLen() > size)) { + if ((!StoreFileInfo.isDelFile(file.getPath())) && (file.getLen() > size)) { count++; } } @@ -776,9 +624,9 @@ private void loadData(Admin admin, BufferedMutator table, TableName tableName, i if (fileNum <= 0) { throw new IllegalArgumentException(); } - for (byte k0 : KEYS) { - byte[] k = new byte[] { k0 }; - for (int i = 0; i < fileNum * rowNumPerFile; i++) { + for (int i = 0; i < fileNum * rowNumPerFile; i++) { + for (byte k0 : KEYS) { + byte[] k = new byte[] { k0 }; byte[] key = Bytes.add(k, Bytes.toBytes(i)); byte[] mobVal = makeDummyData(10 * (i + 1)); Put put = new Put(key); @@ -787,10 +635,10 @@ private void loadData(Admin admin, BufferedMutator table, TableName tableName, i put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal); put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal); table.mutate(put); - if ((i + 1) % rowNumPerFile == 0) { - table.flush(); - admin.flush(tableName); - } + } + if ((i + 1) % rowNumPerFile == 0) { + table.flush(); + admin.flush(tableName); } } } @@ -805,30 +653,30 @@ private void loadData(Admin admin, BufferedMutator table, TableName tableName, P /** * delete the row, family and cell to create the del file */ - private void createDelFile() throws IOException, InterruptedException { + private void createDelFile(Table table, TableName tableName, byte[] family, byte[] qf) + throws IOException, InterruptedException { for (byte k0 : KEYS) { byte[] k = new byte[] { k0 }; // delete a family byte[] key1 = Bytes.add(k, Bytes.toBytes(0)); Delete delete1 = new Delete(key1); - delete1.addFamily(Bytes.toBytes(family1)); - hTable.delete(delete1); + delete1.addFamily(family); + table.delete(delete1); // delete one row byte[] key2 = Bytes.add(k, Bytes.toBytes(2)); Delete delete2 = new Delete(key2); - hTable.delete(delete2); + table.delete(delete2); // delete one cell byte[] key3 = Bytes.add(k, Bytes.toBytes(4)); Delete delete3 = new Delete(key3); - delete3.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1)); - hTable.delete(delete3); - admin.flush(tableName); - List regions = TEST_UTIL.getHBaseCluster().getRegions( - Bytes.toBytes(tableNameAsString)); - for (HRegion region : regions) { - region.waitForFlushesAndCompactions(); - region.compact(true); - } + delete3.addColumn(family, qf); + table.delete(delete3); + } + admin.flush(tableName); + List regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : regions) { + region.waitForFlushesAndCompactions(); + region.compact(true); } } /** @@ -880,7 +728,7 @@ private void assertRefFileNameEqual(String familyName) throws IOException { scan.addFamily(Bytes.toBytes(familyName)); // Do not retrieve the mob data when scanning scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - ResultScanner results = hTable.getScanner(scan); + ResultScanner results = table.getScanner(scan); Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), familyName); List actualFilePaths = new ArrayList<>();