diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index fbd81c9..ef14290 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -68,8 +68,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.mob.compactions.MobCompactor; -import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -81,10 +79,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -92,17 +88,16 @@ @Category(LargeTests.class) public class TestMobCompactor { private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private Configuration conf = null; - private String tableNameAsString; + private static Configuration conf = null; private TableName tableName; private static Connection conn; private BufferedMutator bufMut; - private Table hTable; - private Admin admin; + private Table table; + private static Admin admin; private HTableDescriptor desc; private HColumnDescriptor hcd1; private HColumnDescriptor hcd2; - private FileSystem fs; + private static FileSystem fs; private static final String family1 = "family1"; private static final String family2 = "family2"; private static final String qf1 = "qualifier1"; @@ -127,6 +122,9 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(1); pool = createThreadPool(TEST_UTIL.getConfiguration()); conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + admin = TEST_UTIL.getHBaseAdmin(); } @AfterClass @@ -136,363 +134,128 @@ public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - @Before - public void setUp() throws Exception { - fs = TEST_UTIL.getTestFileSystem(); - conf = TEST_UTIL.getConfiguration(); - long tid = System.currentTimeMillis(); - tableNameAsString = "testMob" + tid; + public void setUp(String tableNameAsString) throws IOException { tableName = TableName.valueOf(tableNameAsString); hcd1 = new HColumnDescriptor(family1); hcd1.setMobEnabled(true); hcd1.setMobThreshold(5); - hcd1.setMaxVersions(4); hcd2 = new HColumnDescriptor(family2); hcd2.setMobEnabled(true); hcd2.setMobThreshold(5); - hcd2.setMaxVersions(4); desc = new HTableDescriptor(tableName); desc.addFamily(hcd1); desc.addFamily(hcd2); - admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(desc, getSplitKeys()); - hTable = conn.getTable(tableName); + table = conn.getTable(tableName); bufMut = conn.getBufferedMutator(tableName); } - @After - public void tearDown() throws Exception { - admin.disableTable(tableName); - admin.deleteTable(tableName); - admin.close(); - hTable.close(); - fs.delete(TEST_UTIL.getDataTestDir(), true); - } - - @Test - public void testCompactionWithoutDelFilesWithNamespace() throws Exception { + @Test(timeout = 300000) + public void testMinorCompaction() throws Exception { resetConf(); + int mergeSize = 5000; + // change the mob compaction merge size + conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); + // create a table with namespace NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build(); - String tableNameAsString = "ns:testCompactionWithoutDelFilesWithNamespace"; + String tableNameAsString = "ns:testMinorCompaction"; admin.createNamespace(namespaceDescriptor); - TableName tableName = TableName.valueOf(tableNameAsString); - HColumnDescriptor hcd1 = new HColumnDescriptor(family1); - hcd1.setMobEnabled(true); - hcd1.setMobThreshold(0); - hcd1.setMaxVersions(4); - HColumnDescriptor hcd2 = new HColumnDescriptor(family2); - hcd2.setMobEnabled(true); - hcd2.setMobThreshold(0); - hcd2.setMaxVersions(4); - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(hcd1); - desc.addFamily(hcd2); - admin.createTable(desc, getSplitKeys()); - BufferedMutator bufMut= conn.getBufferedMutator(tableName); - Table table = conn.getTable(tableName); - + setUp(tableNameAsString); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); int rowNumPerRegion = count * rowNumPerFile; - assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion, + assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, countMobRows(table)); - assertEquals("Before compaction: mob file count", regionNum * count, + assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, + countMobCells(table)); + assertEquals("Before deleting: mob file count", regionNum * count, countFiles(tableName, true, family1)); - assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1)); - MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); - compactor.compact(); + int largeFilesCount = countLargeFiles(mergeSize, tableName, family1); + createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); - assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion, + assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), countMobRows(table)); - assertEquals("After compaction: mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1)); - - table.close(); - admin.disableTable(tableName); - admin.deleteTable(tableName); - admin.deleteNamespace("ns"); - } - - @Test - public void testCompactionWithoutDelFiles() throws Exception { - resetConf(); - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before compaction: mob file count", regionNum * count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1)); - - MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); - compactor.compact(); - - assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("After compaction: mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1)); - } - - @Test - public void testCompactionWithoutDelFilesAndWithEncryption() throws Exception { - resetConf(); - Configuration conf = TEST_UTIL.getConfiguration(); - SecureRandom rng = new SecureRandom(); - byte[] keyBytes = new byte[AES.KEY_LENGTH]; - rng.nextBytes(keyBytes); - String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); - Key cfKey = new SecretKeySpec(keyBytes, algorithm); - byte[] encryptionKey = EncryptionUtil.wrapKey(conf, - conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey); - String tableNameAsString = "testCompactionWithoutDelFilesAndWithEncryption"; - TableName tableName = TableName.valueOf(tableNameAsString); - HTableDescriptor desc = new HTableDescriptor(tableName); - HColumnDescriptor hcd = new HColumnDescriptor(family1); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0); - hcd.setMaxVersions(4); - hcd.setEncryptionType(algorithm); - hcd.setEncryptionKey(encryptionKey); - HColumnDescriptor hcd2 = new HColumnDescriptor(family2); - hcd2.setMobEnabled(true); - hcd2.setMobThreshold(0); - hcd2.setMaxVersions(4); - desc.addFamily(hcd); - desc.addFamily(hcd2); - admin.createTable(desc, getSplitKeys()); - Table hTable = conn.getTable(tableName); - BufferedMutator bufMut = conn.getBufferedMutator(tableName); - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before compaction: mob file count", regionNum * count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1)); - - MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd, pool); - compactor.compact(); - - assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("After compaction: mob file count", regionNum, + assertEquals("Before compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); + assertEquals("Before compaction: family1 mob file count", regionNum * count, countFiles(tableName, true, family1)); - assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1)); - Assert.assertTrue(verifyEncryption(tableName, family1)); - } - - @Test - public void testCompactionWithDelFiles() throws Exception { - resetConf(); - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before deleting: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("Before compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); - assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - - // do the mob file compaction - MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); - compactor.compact(); - - assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("After compaction: family1 mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("After compaction: family1 del file count", 0, countFiles(tableName, false, family1)); - assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - assertRefFileNameEqual(family1); - } - - @Test - public void testCompactionWithDelFilesAndNotMergeAllFiles() throws Exception { - resetConf(); - int mergeSize = 5000; - // change the mob compaction merge size - conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); - - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: mob file count", regionNum * count, - countFiles(tableName, true, family1)); - - int largeFilesCount = countLargeFiles(mergeSize, family1); - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); // do the mob file compaction MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); compactor.compact(); - assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); + assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), + countMobRows(table)); + assertEquals("After compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); // After the compaction, the files smaller than the mob compaction merge size // is merge to one file assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum, - countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + countFiles(tableName, true, family1)); + assertEquals("After compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); - assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - } - - @Test - public void testCompactionWithDelFilesAndWithSmallCompactionBatchSize() throws Exception { - resetConf(); - int batchSize = 2; - conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize); - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob row count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before deleting: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); - assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - - // do the mob compaction - MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); - compactor.compact(); - - assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize), - countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("After compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); } - @Test + @Test(timeout = 300000) public void testCompactionWithHFileLink() throws IOException, InterruptedException { resetConf(); + String tableNameAsString = "testCompactionWithHFileLink"; + setUp(tableNameAsString); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; + int rowNumPerRegion = count * rowNumPerFile; long tid = System.currentTimeMillis(); byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid); // take a snapshot admin.snapshot(snapshotName1, tableName); - createDelFile(); + createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), + countMobRows(table)); + assertEquals("Before compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); + assertEquals("Before compaction: family1 mob file count", regionNum * count, + countFiles(tableName, true, family1)); + assertEquals("Before compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); // do the mob compaction MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool); compactor.compact(); - assertEquals("After first compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After first compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); + assertEquals("After first compaction: mob rows count", regionNum + * (rowNumPerRegion - delRowNum), countMobRows(table)); + assertEquals("After first compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After first compaction: family1 mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After first compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + countFiles(tableName, true, family1)); + assertEquals("After first compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After first compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After first compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1)); assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2)); @@ -501,33 +264,32 @@ public void testCompactionWithHFileLink() throws IOException, InterruptedExcepti admin.restoreSnapshot(snapshotName1); admin.enableTable(tableName); - assertEquals("After restoring snapshot: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("After restoring snapshot: mob cells count", - regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable)); - assertEquals("After restoring snapshot: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("After restoring snapshot: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion, + countMobRows(table)); + assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow + * rowNumPerRegion, countMobCells(table)); + assertEquals("After restoring snapshot: family1 mob file count", regionNum * count, + countFiles(tableName, true, family1)); + assertEquals("After restoring snapshot: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After restoring snapshot: family1 del file count", 0, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("After restoring snapshot: family2 del file count", 0, - countFiles(tableName, false, family2)); - assertEquals("After restoring snapshot: family1 hfilelink count", regionNum*count, - countHFileLinks(family1)); - assertEquals("After restoring snapshot: family2 hfilelink count", 0, - countHFileLinks(family2)); + countFiles(tableName, false, family2)); + assertEquals("After restoring snapshot: family1 hfilelink count", regionNum * count, + countHFileLinks(family1)); + assertEquals("After restoring snapshot: family2 hfilelink count", 0, countHFileLinks(family2)); compactor.compact(); - assertEquals("After second compaction: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("After second compaction: mob cells count", - regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable)); + assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion, + countMobRows(table)); + assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow + * rowNumPerRegion, countMobCells(table)); assertEquals("After second compaction: family1 mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After second compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + countFiles(tableName, true, family1)); + assertEquals("After second compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("After second compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After second compaction: family2 del file count", 0, @@ -537,106 +299,86 @@ public void testCompactionWithHFileLink() throws IOException, InterruptedExcepti assertRefFileNameEqual(family1); } - @Test - public void testCompactionFromAdmin() throws Exception { + @Test(timeout = 300000) + public void testMajorCompactionFromAdmin() throws Exception { + resetConf(); + int mergeSize = 5000; + // change the mob compaction merge size + conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); + String tableNameAsString = "testMajorCompactionFromAdmin"; + SecureRandom rng = new SecureRandom(); + byte[] keyBytes = new byte[AES.KEY_LENGTH]; + rng.nextBytes(keyBytes); + String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Key cfKey = new SecretKeySpec(keyBytes, algorithm); + byte[] encryptionKey = EncryptionUtil.wrapKey(conf, + conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey); + TableName tableName = TableName.valueOf(tableNameAsString); + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor hcd1 = new HColumnDescriptor(family1); + hcd1.setMobEnabled(true); + hcd1.setMobThreshold(0); + hcd1.setEncryptionType(algorithm); + hcd1.setEncryptionKey(encryptionKey); + HColumnDescriptor hcd2 = new HColumnDescriptor(family2); + hcd2.setMobEnabled(true); + hcd2.setMobThreshold(0); + desc.addFamily(hcd1); + desc.addFamily(hcd2); + admin.createTable(desc, getSplitKeys()); + Table table = conn.getTable(tableName); + BufferedMutator bufMut = conn.getBufferedMutator(tableName); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; + int rowNumPerRegion = count * rowNumPerFile; - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before deleting: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, + countMobRows(table)); + assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, + countMobCells(table)); + assertEquals("Before deleting: mob file count", regionNum * count, + countFiles(tableName, true, family1)); - createDelFile(); + createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 file count", regionNum*count, - countFiles(tableName, true, family2)); + assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), + countMobRows(table)); + assertEquals("Before compaction: mob cells count", regionNum + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); + assertEquals("Before compaction: family1 mob file count", regionNum * count, + countFiles(tableName, true, family1)); + assertEquals("Before compaction: family2 mob file count", regionNum * count, + countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); + countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + countFiles(tableName, false, family2)); - int largeFilesCount = countLargeFiles(5000, family1); - // do the mob compaction - admin.compact(tableName, hcd1.getName(), Admin.CompactType.MOB); + // do the major mob compaction, it will force all files to compaction + admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB); waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(hTable)); + countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum - * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(hTable)); - assertEquals("After compaction: family1 mob file count", regionNum + largeFilesCount, + * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); + assertEquals("After compaction: family1 mob file count", regionNum, countFiles(tableName, true, family1)); assertEquals("After compaction: family2 mob file count", regionNum * count, countFiles(tableName, true, family2)); - assertEquals("After compaction: family1 del file count", regionNum, + assertEquals("After compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After compaction: family2 del file count", regionNum, countFiles(tableName, false, family2)); - assertRefFileNameEqual(family1); - } - - @Test - public void testMajorCompactionFromAdmin() throws Exception { - int count = 4; - // generate mob files - loadData(admin, bufMut, tableName, count, rowNumPerFile); - int rowNumPerRegion = count*rowNumPerFile; - - assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion, - countMobRows(hTable)); - assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion, - countMobCells(hTable)); - assertEquals("Before deleting: mob file count", regionNum*count, - countFiles(tableName, true, family1)); - - createDelFile(); - - assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("Before compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("Before compaction: family1 mob file count", regionNum*count, - countFiles(tableName, true, family1)); - assertEquals("Before compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("Before compaction: family1 del file count", regionNum, - countFiles(tableName, false, family1)); - assertEquals("Before compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); - - // do the major mob compaction, it will force all files to compaction - admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB); - - waitUntilMobCompactionFinished(tableName); - assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), - countMobRows(hTable)); - assertEquals("After compaction: mob cells count", - regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable)); - assertEquals("After compaction: family1 mob file count", regionNum, - countFiles(tableName, true, family1)); - assertEquals("After compaction: family2 mob file count", regionNum*count, - countFiles(tableName, true, family2)); - assertEquals("After compaction: family1 del file count", 0, - countFiles(tableName, false, family1)); - assertEquals("After compaction: family2 del file count", regionNum, - countFiles(tableName, false, family2)); + Assert.assertTrue(verifyEncryption(tableName, family1)); + table.close(); } - @Test + @Test(timeout = 300000) public void testScannerOnBulkLoadRefHFiles() throws Exception { + resetConf(); + setUp("testScannerOnBulkLoadRefHFiles"); long ts = EnvironmentEdgeManager.currentTime(); byte[] key0 = Bytes.toBytes("k0"); byte[] key1 = Bytes.toBytes("k1"); @@ -653,7 +395,7 @@ public void testScannerOnBulkLoadRefHFiles() throws Exception { loadData(admin, bufMut, tableName, new Put[] { put0, put1 }); // read the latest cell of key0. Get get = new Get(key0); - Result result = hTable.get(get); + Result result = table.get(get); Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob value of k0", newValue0, Bytes.toString(CellUtil.cloneValue(cell))); @@ -661,18 +403,18 @@ public void testScannerOnBulkLoadRefHFiles() throws Exception { waitUntilMobCompactionFinished(tableName); // read the latest cell of key0, the cell seqId in bulk loaded file is not reset in the // scanner. The cell that has "new" value is still visible. - result = hTable.get(get); + result = table.get(get); cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); assertEquals("After compaction: mob value of k0", newValue0, Bytes.toString(CellUtil.cloneValue(cell))); // read the ref cell, not read further to the mob cell. get = new Get(key1); get.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(true)); - result = hTable.get(get); + result = table.get(get); cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); // the ref name is the new file - Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), - tableName), hcd1.getNameAsString()); + Path mobFamilyPath = new Path( + MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), hcd1.getNameAsString()); List paths = new ArrayList(); if (fs.exists(mobFamilyPath)) { FileStatus[] files = fs.listStatus(mobFamilyPath); @@ -687,8 +429,10 @@ public void testScannerOnBulkLoadRefHFiles() throws Exception { .getName()); } - @Test + @Test(timeout = 300000) public void testScannerAfterCompactions() throws Exception { + resetConf(); + setUp("testScannerAfterCompactions"); long ts = EnvironmentEdgeManager.currentTime(); byte[] key0 = Bytes.toBytes("k0"); byte[] key1 = Bytes.toBytes("k1"); @@ -709,7 +453,7 @@ public void testScannerAfterCompactions() throws Exception { waitUntilMobCompactionFinished(tableName); // read the latest cell of key1. Get get = new Get(key1); - Result result = hTable.get(get); + Result result = table.get(get); Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); assertEquals("After compaction: mob value", "new", Bytes.toString(CellUtil.cloneValue(cell))); } @@ -852,19 +596,19 @@ private int countHFileLinks(String familyName) throws IOException { /** * Gets the number of files. * @param size the size of the file + * @param tableName the current table name * @param familyName the family name * @return the number of files large than the size */ - private int countLargeFiles(int size, String familyName) throws IOException { - Path mobDirPath = MobUtils.getMobFamilyPath( - MobUtils.getMobRegionPath(conf, tableName), familyName); + private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException { + Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName), + familyName); int count = 0; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); for (FileStatus file : files) { // ignore the del files in the mob path - if ((!StoreFileInfo.isDelFile(file.getPath())) - && (file.getLen() > size)) { + if ((!StoreFileInfo.isDelFile(file.getPath())) && (file.getLen() > size)) { count++; } } @@ -909,26 +653,26 @@ private void loadData(Admin admin, BufferedMutator table, TableName tableName, P /** * delete the row, family and cell to create the del file */ - private void createDelFile() throws IOException, InterruptedException { + private void createDelFile(Table table, TableName tableName, byte[] family, byte[] qf) + throws IOException, InterruptedException { for (byte k0 : KEYS) { byte[] k = new byte[] { k0 }; // delete a family byte[] key1 = Bytes.add(k, Bytes.toBytes(0)); Delete delete1 = new Delete(key1); - delete1.addFamily(Bytes.toBytes(family1)); - hTable.delete(delete1); + delete1.addFamily(family); + table.delete(delete1); // delete one row byte[] key2 = Bytes.add(k, Bytes.toBytes(2)); Delete delete2 = new Delete(key2); - hTable.delete(delete2); + table.delete(delete2); // delete one cell byte[] key3 = Bytes.add(k, Bytes.toBytes(4)); Delete delete3 = new Delete(key3); - delete3.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1)); - hTable.delete(delete3); + delete3.addColumn(family, qf); + table.delete(delete3); admin.flush(tableName); - List regions = TEST_UTIL.getHBaseCluster().getRegions( - Bytes.toBytes(tableNameAsString)); + List regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); for (HRegion region : regions) { region.waitForFlushesAndCompactions(); region.compact(true); @@ -984,7 +728,7 @@ private void assertRefFileNameEqual(String familyName) throws IOException { scan.addFamily(Bytes.toBytes(familyName)); // Do not retrieve the mob data when scanning scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - ResultScanner results = hTable.getScanner(scan); + ResultScanner results = table.getScanner(scan); Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), familyName); List actualFilePaths = new ArrayList<>(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/mapreduce/TestMobSweeper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/mapreduce/TestMobSweeper.java deleted file mode 100644 index 6e6eac9..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/mapreduce/TestMobSweeper.java +++ /dev/null @@ -1,368 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.mob.mapreduce; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.security.Key; -import java.security.SecureRandom; -import java.util.Random; -import java.util.Set; -import java.util.TreeSet; - -import javax.crypto.spec.SecretKeySpec; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; -import org.apache.hadoop.hbase.io.crypto.aes.AES; -import org.apache.hadoop.hbase.io.hfile.CacheConfig; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.regionserver.BloomType; -import org.apache.hadoop.hbase.regionserver.StoreFile; -import org.apache.hadoop.hbase.security.EncryptionUtil; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(MediumTests.class) -public class TestMobSweeper { - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private TableName tableName; - private final static String row = "row_"; - private final static String family = "family"; - private final static String column = "column"; - private static Table table; - private static BufferedMutator bufMut; - private static Admin admin; - - private Random random = new Random(); - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0); - TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true); - // avoid major compactions - TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 15); - // avoid major compactions - TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.max", 30); - TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3); - TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, - KeyProviderForTesting.class.getName()); - TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); - - TEST_UTIL.startMiniCluster(); - - TEST_UTIL.startMiniMapReduceCluster(); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - TEST_UTIL.shutdownMiniMapReduceCluster(); - } - - @Before - public void setUp() throws Exception { - long tid = System.currentTimeMillis(); - tableName = TableName.valueOf("testSweeper" + tid); - HTableDescriptor desc = new HTableDescriptor(tableName); - HColumnDescriptor hcd = new HColumnDescriptor(family); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0); - hcd.setMaxVersions(4); - desc.addFamily(hcd); - - admin = TEST_UTIL.getHBaseAdmin(); - admin.createTable(desc); - Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - table = c.getTable(tableName); - bufMut = c.getBufferedMutator(tableName); - } - - @After - public void tearDown() throws Exception { - admin.disableTable(tableName); - admin.deleteTable(tableName); - admin.close(); - } - - private Path getMobFamilyPath(Configuration conf, TableName tableName, - String familyName) { - Path p = new Path(MobUtils.getMobRegionPath(conf, tableName), - familyName); - return p; - } - - private String mergeString(Set set) { - StringBuilder sb = new StringBuilder(); - for (String s : set) - sb.append(s); - return sb.toString(); - } - - private void generateMobTable(Admin admin, BufferedMutator table, TableName tableName, int count, - int flushStep) throws IOException, InterruptedException { - if (count <= 0 || flushStep <= 0) - return; - int index = 0; - for (int i = 0; i < count; i++) { - byte[] mobVal = new byte[101*1024]; - random.nextBytes(mobVal); - - Put put = new Put(Bytes.toBytes(row + i)); - put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), mobVal); - table.mutate(put); - if (index++ % flushStep == 0) { - table.flush(); - admin.flush(tableName); - } - } - table.flush(); - admin.flush(tableName); - } - - @Test - public void testSweeper() throws Exception { - int count = 10; - //create table and generate 10 mob files - generateMobTable(admin, bufMut, tableName, count, 1); - //get mob files - Path mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family); - FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); - // mobFileSet0 stores the original mob files - TreeSet mobFilesSet = new TreeSet(); - for (FileStatus status : fileStatuses) { - mobFilesSet.add(status.getPath().getName()); - } - - //scan the table, retreive the references - Scan scan = new Scan(); - scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE)); - ResultScanner rs = table.getScanner(scan); - TreeSet mobFilesScanned = new TreeSet(); - for (Result res : rs) { - byte[] valueBytes = res.getValue(Bytes.toBytes(family), - Bytes.toBytes(column)); - mobFilesScanned.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT, - valueBytes.length - Bytes.SIZEOF_INT)); - } - //there should be 10 mob files - assertEquals(10, mobFilesScanned.size()); - //check if we store the correct reference of mob files - assertEquals(mergeString(mobFilesSet), mergeString(mobFilesScanned)); - - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setLong(SweepJob.MOB_SWEEP_JOB_DELAY, 24 * 60 * 60 * 1000); - - String[] args = new String[2]; - args[0] = tableName.getNameAsString(); - args[1] = family; - assertEquals(0, ToolRunner.run(conf, new Sweeper(), args)); - - mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family); - fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); - mobFilesSet = new TreeSet(); - for (FileStatus status : fileStatuses) { - mobFilesSet.add(status.getPath().getName()); - } - assertEquals(10, mobFilesSet.size()); - - scan = new Scan(); - scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE)); - rs = table.getScanner(scan); - TreeSet mobFilesScannedAfterJob = new TreeSet(); - for (Result res : rs) { - byte[] valueBytes = res.getValue(Bytes.toBytes(family), Bytes.toBytes( - column)); - mobFilesScannedAfterJob.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT, - valueBytes.length - Bytes.SIZEOF_INT)); - } - assertEquals(10, mobFilesScannedAfterJob.size()); - - fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); - mobFilesSet = new TreeSet(); - for (FileStatus status : fileStatuses) { - mobFilesSet.add(status.getPath().getName()); - } - assertEquals(10, mobFilesSet.size()); - assertEquals(true, mobFilesScannedAfterJob.iterator().next() - .equalsIgnoreCase(mobFilesSet.iterator().next())); - } - - private void testCompactionDelaySweeperInternal(Table table, BufferedMutator bufMut, - TableName tableName, boolean encrypted) throws Exception { - int count = 10; - //create table and generate 10 mob files - generateMobTable(admin, bufMut, tableName, count, 1); - //get mob files - Path mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family); - FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); - // mobFileSet0 stores the orignal mob files - TreeSet mobFilesSet = new TreeSet(); - for (FileStatus status : fileStatuses) { - mobFilesSet.add(status.getPath().getName()); - } - - //scan the table, retreive the references - Scan scan = new Scan(); - scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE)); - ResultScanner rs = table.getScanner(scan); - TreeSet mobFilesScanned = new TreeSet(); - for (Result res : rs) { - byte[] valueBytes = res.getValue(Bytes.toBytes(family), - Bytes.toBytes(column)); - mobFilesScanned.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT, - valueBytes.length - Bytes.SIZEOF_INT)); - } - //there should be 10 mob files - assertEquals(10, mobFilesScanned.size()); - //check if we store the correct reference of mob files - assertEquals(mergeString(mobFilesSet), mergeString(mobFilesScanned)); - - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setLong(SweepJob.MOB_SWEEP_JOB_DELAY, 0); - String[] args = new String[2]; - args[0] = tableName.getNameAsString(); - args[1] = family; - assertEquals(0, ToolRunner.run(conf, new Sweeper(), args)); - - mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family); - fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); - mobFilesSet = new TreeSet(); - for (FileStatus status : fileStatuses) { - mobFilesSet.add(status.getPath().getName()); - } - assertEquals(1, mobFilesSet.size()); - - scan = new Scan(); - scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE)); - rs = table.getScanner(scan); - TreeSet mobFilesScannedAfterJob = new TreeSet(); - for (Result res : rs) { - byte[] valueBytes = res.getValue(Bytes.toBytes(family), Bytes.toBytes( - column)); - mobFilesScannedAfterJob.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT, - valueBytes.length - Bytes.SIZEOF_INT)); - } - assertEquals(1, mobFilesScannedAfterJob.size()); - - fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); - Path lastFilePath = null; - mobFilesSet = new TreeSet(); - for (FileStatus status : fileStatuses) { - mobFilesSet.add(status.getPath().getName()); - lastFilePath = status.getPath(); - } - assertEquals(1, mobFilesSet.size()); - assertEquals(true, mobFilesScannedAfterJob.iterator().next() - .equalsIgnoreCase(mobFilesSet.iterator().next())); - if (encrypted) { - // assert the encryption context - CacheConfig cacheConf = new CacheConfig(conf); - StoreFile sf = new StoreFile(TEST_UTIL.getTestFileSystem(), lastFilePath, conf, cacheConf, - BloomType.NONE); - HFile.Reader reader = sf.createReader().getHFileReader(); - byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); - Assert.assertTrue(null != encryptionKey); - Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() - .equals(HConstants.CIPHER_AES)); - } - } - - @Test - public void testCompactionDelaySweeperWithEncryption() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - SecureRandom rng = new SecureRandom(); - byte[] keyBytes = new byte[AES.KEY_LENGTH]; - rng.nextBytes(keyBytes); - String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); - Key cfKey = new SecretKeySpec(keyBytes, algorithm); - byte[] encryptionKey = EncryptionUtil.wrapKey(conf, - conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey); - String tableNameAsString = "testCompactionDelaySweeperWithEncryption"; - TableName tableName = TableName.valueOf(tableNameAsString); - HTableDescriptor desc = new HTableDescriptor(tableName); - HColumnDescriptor hcd = new HColumnDescriptor(family); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0); - hcd.setMaxVersions(4); - hcd.setEncryptionType(algorithm); - hcd.setEncryptionKey(encryptionKey); - desc.addFamily(hcd); - admin.createTable(desc); - Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - BufferedMutator bufMut = c.getBufferedMutator(tableName); - Table table = c.getTable(tableName); - testCompactionDelaySweeperInternal(table, bufMut, tableName, true); - table.close(); - admin.disableTable(tableName); - admin.deleteTable(tableName); - } - - @Test - public void testCompactionDelaySweeper() throws Exception { - testCompactionDelaySweeperInternal(table, bufMut, tableName, false); - } - - @Test - public void testCompactionDelaySweeperWithNamespace() throws Exception { - // create a table with namespace - NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build(); - admin.createNamespace(namespaceDescriptor); - String tableNameAsString = "ns:testSweeperWithNamespace"; - TableName tableName = TableName.valueOf(tableNameAsString); - HTableDescriptor desc = new HTableDescriptor(tableName); - HColumnDescriptor hcd = new HColumnDescriptor(family); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0); - hcd.setMaxVersions(4); - desc.addFamily(hcd); - admin.createTable(desc); - Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - BufferedMutator bufMut = c.getBufferedMutator(tableName); - Table table = c.getTable(tableName); - testCompactionDelaySweeperInternal(table, bufMut, tableName, false); - table.close(); - admin.disableTable(tableName); - admin.deleteTable(tableName); - admin.deleteNamespace("ns"); - } -}