From ad39cd17e1bcf3d5f5fe7d8e10eedf49c297ee1b Mon Sep 17 00:00:00 2001 From: Artem Ervits Date: Fri, 19 Oct 2018 10:42:41 -0400 Subject: [PATCH] HBASE-21194 Add TestCopyTable which exercises MOB feature --- .../hadoop/hbase/mapreduce/TestCopyTable.java | 129 ++++++++++++++++++--- 1 file changed, 113 insertions(+), 16 deletions(-) diff --git hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java index 6f416710e1..5d5e084c1c 100644 --- hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java +++ hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java @@ -24,16 +24,22 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.PrintStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.util.Bytes; @@ -121,9 +127,69 @@ public class TestCopyTable { } } + private void doCopyTableTestWithMob(boolean bulkload) throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("mob"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + + ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); + + cfd.setMobEnabled(true); + cfd.setMobThreshold(10); + TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1) + .setColumnFamily(cfd.build()) + .build(); + TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2) + .setColumnFamily(cfd.build()) + .build(); + + try (Table t1 = TEST_UTIL.createTable(desc1, null); + Table t2 = TEST_UTIL.createTable(desc2, null);) { + + // put rows into the first table + for (int i = 0; i < 10; i++) { + Put p = new Put(Bytes.toBytes("row" + i)); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + } + + CopyTable copy = new CopyTable(); + + int code; + if (bulkload) { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + "--bulkload", tableName1.getNameAsString() }); + } else { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + tableName1.getNameAsString() }); + } + assertEquals("copy job failed", 0, code); + + // verify the data was copied into table 2 + for (int i = 0; i < 10; i++) { + Get g = new Get(Bytes.toBytes("row" + i)); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + assertEquals("compare row values", t1.getDescriptor().getValue("row" + i), t2.getDescriptor().getValue("row" + i)); + } + + assertEquals("compare mob rows after copy", countMobRows(t1), countMobRows(t2)); + assertEquals("compare count of all values between two tables", t1.getDescriptor().getValues().size(), + t2.getDescriptor().getValues().size()); + + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } + } + /** * Simple end-to-end test - * @throws Exception + * @throws java.lang.Exception */ @Test public void testCopyTable() throws Exception { @@ -132,33 +198,52 @@ public class TestCopyTable { /** * Simple end-to-end test with bulkload. + * @throws java.lang.Exception */ @Test public void testCopyTableWithBulkload() throws Exception { doCopyTableTest(true); } + /** + * Simple end-to-end test on table with MOB + * @throws java.lang.Exception + */ + @Test + public void testCopyTableWithMob() throws Exception { + doCopyTableTestWithMob(false); + } + + /** + * Simple end-to-end test with bulkload on table with MOB. + * @throws java.lang.Exception + */ + @Test + public void testCopyTableWithBulkloadWithMob() throws Exception { + doCopyTableTestWithMob(true); + } + @Test public void testStartStopRow() throws Exception { final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); - final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0"); - final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1"); - final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2"); + final byte[] row0 = Bytes.toBytesBinary("\\x01row0"); + final byte[] row1 = Bytes.toBytesBinary("\\x01row1"); + final byte[] row2 = Bytes.toBytesBinary("\\x01row2"); - Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); - Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { // put rows into the first table - Put p = new Put(ROW0); + Put p = new Put(row0); p.addColumn(FAMILY, COLUMN1, COLUMN1); t1.put(p); - p = new Put(ROW1); + p = new Put(row1); p.addColumn(FAMILY, COLUMN1, COLUMN1); t1.put(p); - p = new Put(ROW2); + p = new Put(row2); p.addColumn(FAMILY, COLUMN1, COLUMN1); t1.put(p); @@ -171,23 +256,23 @@ public class TestCopyTable { // verify the data was copied into table 2 // row1 exist, row0, row2 do not exist - Get g = new Get(ROW1); + Get g = new Get(row1); Result r = t2.get(g); assertEquals(1, r.size()); assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); - g = new Get(ROW0); + g = new Get(row0); r = t2.get(g); assertEquals(0, r.size()); - g = new Get(ROW2); + g = new Get(row2); r = t2.get(g); assertEquals(0, r.size()); - t1.close(); - t2.close(); - TEST_UTIL.deleteTable(tableName1); - TEST_UTIL.deleteTable(tableName2); + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } } /** @@ -265,4 +350,16 @@ public class TestCopyTable { args); return status == 0; } + + /** + * Gets the number of rows in the given table. + * @param table to get the scanner + * @return the number of rows + */ + private int countMobRows(final Table table) throws IOException { + Scan scan = new Scan(); + // Do not retrieve the mob data when scanning + scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); + return TEST_UTIL.countRows(table, scan); + } } -- 2.16.2