From 5fe07cde67dc7c595912a0a0554aaa848d345994 Mon Sep 17 00:00:00 2001 From: Artem Ervits Date: Thu, 18 Oct 2018 16:51:10 -0400 Subject: [PATCH] HBASE-21194 Add TestCopyTable which exercises MOB feature --- .../hadoop/hbase/mapreduce/TestCopyTable.java | 108 ++++++++++++++++++--- 1 file changed, 92 insertions(+), 16 deletions(-) diff --git hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java index 6f416710e1..af5d856ef5 100644 --- hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java +++ hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java @@ -30,10 +30,13 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.util.Bytes; @@ -121,9 +124,63 @@ public class TestCopyTable { } } + private void doCopyTableTestWithMob(boolean bulkload) throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("mob"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + + ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); + + cfd.setMobEnabled(true); + cfd.setMobThreshold(102400); + TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1) + .setColumnFamily(cfd.build()) + .build(); + TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2) + .setColumnFamily(cfd.build()) + .build(); + + try (Table t1 = TEST_UTIL.createTable(desc1, null); + Table t2 = TEST_UTIL.createTable(desc2, null);) { + + // put rows into the first table + for (int i = 0; i < 10; i++) { + Put p = new Put(Bytes.toBytes("row" + i)); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + } + + CopyTable copy = new CopyTable(); + + int code; + if (bulkload) { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + "--bulkload", tableName1.getNameAsString() }); + } else { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + tableName1.getNameAsString() }); + } + assertEquals("copy job failed", 0, code); + + // verify the data was copied into table 2 + for (int i = 0; i < 10; i++) { + Get g = new Get(Bytes.toBytes("row" + i)); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + } + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } + } + /** * Simple end-to-end test - * @throws Exception + * @throws java.lang.Exception */ @Test public void testCopyTable() throws Exception { @@ -132,11 +189,30 @@ public class TestCopyTable { /** * Simple end-to-end test with bulkload. + * @throws java.lang.Exception */ @Test public void testCopyTableWithBulkload() throws Exception { doCopyTableTest(true); } + + /** + * Simple end-to-end test on table with MOB + * @throws java.lang.Exception + */ + @Test + public void testCopyTableWithMob() throws Exception { + doCopyTableTestWithMob(false); + } + + /** + * Simple end-to-end test with bulkload on table with MOB. + * @throws java.lang.Exception + */ + @Test + public void testCopyTableWithBulkloadWithMob() throws Exception { + doCopyTableTestWithMob(true); + } @Test public void testStartStopRow() throws Exception { @@ -144,21 +220,21 @@ public class TestCopyTable { final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); - final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0"); - final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1"); - final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2"); + final byte[] row0 = Bytes.toBytesBinary("\\x01row0"); + final byte[] row1 = Bytes.toBytesBinary("\\x01row1"); + final byte[] row2 = Bytes.toBytesBinary("\\x01row2"); - Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); - Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { // put rows into the first table - Put p = new Put(ROW0); + Put p = new Put(row0); p.addColumn(FAMILY, COLUMN1, COLUMN1); t1.put(p); - p = new Put(ROW1); + p = new Put(row1); p.addColumn(FAMILY, COLUMN1, COLUMN1); t1.put(p); - p = new Put(ROW2); + p = new Put(row2); p.addColumn(FAMILY, COLUMN1, COLUMN1); t1.put(p); @@ -171,23 +247,23 @@ public class TestCopyTable { // verify the data was copied into table 2 // row1 exist, row0, row2 do not exist - Get g = new Get(ROW1); + Get g = new Get(row1); Result r = t2.get(g); assertEquals(1, r.size()); assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); - g = new Get(ROW0); + g = new Get(row0); r = t2.get(g); assertEquals(0, r.size()); - g = new Get(ROW2); + g = new Get(row2); r = t2.get(g); assertEquals(0, r.size()); - t1.close(); - t2.close(); - TEST_UTIL.deleteTable(tableName1); - TEST_UTIL.deleteTable(tableName2); + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } } /** -- 2.16.2