From cf020daa1a1873362e796488b255e37462301933 Mon Sep 17 00:00:00 2001 From: Artem Ervits Date: Fri, 19 Oct 2018 13:06:38 -0400 Subject: [PATCH] HBASE-21194 Add TestCopyTable which exercises MOB feature --- .../hadoop/hbase/mapreduce/TestCopyTable.java | 170 +++++++++++++++------ .../apache/hadoop/hbase/HBaseTestingUtility.java | 6 +- .../org/apache/hadoop/hbase/mob/MobTestUtil.java | 13 ++ .../hbase/mob/compactions/TestMobCompactor.java | 33 ++-- 4 files changed, 155 insertions(+), 67 deletions(-) diff --git hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java index 6f416710e1..885e116843 100644 --- hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java +++ hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java @@ -30,15 +30,19 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.LauncherSecurityManager; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.hbase.mob.MobTestUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -121,9 +125,73 @@ public class TestCopyTable { } } + private void doCopyTableTestWithMob(boolean bulkload) throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("mob"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + + ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); + + cfd.setMobEnabled(true); + cfd.setMobThreshold(5); + TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1) + .setColumnFamily(cfd.build()) + .build(); + TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2) + .setColumnFamily(cfd.build()) + .build(); + + try (Table t1 = TEST_UTIL.createTable(desc1, null); + Table t2 = TEST_UTIL.createTable(desc2, null);) { + + // put rows into the first table + for (int i = 0; i < 10; i++) { + Put p = new Put(Bytes.toBytes("row" + i)); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + } + + CopyTable copy = new CopyTable(); + + int code; + if (bulkload) { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + "--bulkload", tableName1.getNameAsString() }); + } else { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + tableName1.getNameAsString() }); + } + assertEquals("copy job failed", 0, code); + + // verify the data was copied into table 2 + for (int i = 0; i < 10; i++) { + Get g = new Get(Bytes.toBytes("row" + i)); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + assertEquals("compare row values between two tables", t1.getDescriptor().getValue("row" + i), + t2.getDescriptor().getValue("row" + i)); + } + + assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1), + MobTestUtil.countMobRows(t2)); + assertEquals("compare count of mob row values between two tables", + t1.getDescriptor().getValues().size(), + t2.getDescriptor().getValues().size()); + assertTrue("if the assertion fails, the message should give the mob row count (0)", + MobTestUtil.countMobRows(t2) > 0); + + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } + } + /** * Simple end-to-end test - * @throws Exception */ @Test public void testCopyTable() throws Exception { @@ -138,56 +206,72 @@ public class TestCopyTable { doCopyTableTest(true); } + /** + * Simple end-to-end test on table with MOB + */ + @Test + public void testCopyTableWithMob() throws Exception { + doCopyTableTestWithMob(false); + } + + /** + * Simple end-to-end test with bulkload on table with MOB. + */ + @Test + public void testCopyTableWithBulkloadWithMob() throws Exception { + doCopyTableTestWithMob(true); + } + @Test public void testStartStopRow() throws Exception { final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); - final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0"); - final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1"); - final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2"); - - Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); - Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); - - // put rows into the first table - Put p = new Put(ROW0); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); - p = new Put(ROW1); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); - p = new Put(ROW2); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); + final byte[] row0 = Bytes.toBytesBinary("\\x01row0"); + final byte[] row1 = Bytes.toBytesBinary("\\x01row1"); + final byte[] row2 = Bytes.toBytesBinary("\\x01row2"); - CopyTable copy = new CopyTable(); - assertEquals( - 0, + try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { + + // put rows into the first table + Put p = new Put(row0); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + p = new Put(row1); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + p = new Put(row2); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + + CopyTable copy = new CopyTable(); + assertEquals( + 0, ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), - copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1", - "--stoprow=\\x01row2", tableName1.getNameAsString() })); - - // verify the data was copied into table 2 - // row1 exist, row0, row2 do not exist - Get g = new Get(ROW1); - Result r = t2.get(g); - assertEquals(1, r.size()); - assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); - - g = new Get(ROW0); - r = t2.get(g); - assertEquals(0, r.size()); - - g = new Get(ROW2); - r = t2.get(g); - assertEquals(0, r.size()); - - t1.close(); - t2.close(); - TEST_UTIL.deleteTable(tableName1); - TEST_UTIL.deleteTable(tableName2); + copy, new String[]{"--new.name=" + tableName2, "--startrow=\\x01row1", + "--stoprow=\\x01row2", tableName1.getNameAsString()})); + + // verify the data was copied into table 2 + // row1 exist, row0, row2 do not exist + Get g = new Get(row1); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + + g = new Get(row0); + r = t2.get(g); + assertEquals(0, r.size()); + + g = new Get(row2); + r = t2.get(g); + assertEquals(0, r.size()); + + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } } /** diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index e2e4aec00b..7f5e11a11a 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2309,12 +2309,14 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * Return the number of rows in the given table. + * @param table to count rows + * @return count of rows */ - public int countRows(final Table table) throws IOException { + public static int countRows(final Table table) throws IOException { return countRows(table, new Scan()); } - public int countRows(final Table table, final Scan scan) throws IOException { + public static int countRows(final Table table, final Scan scan) throws IOException { try (ResultScanner results = table.getScanner(scan)) { int count = 0; while (results.next() != null) { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java index 76cb664596..053ff90e1f 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java @@ -24,6 +24,7 @@ import java.util.Random; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -104,4 +105,16 @@ public class MobTestUtil { results.close(); Assert.assertEquals(expectedCount, count); } + + /** + * Gets the number of rows in the given table. + * @param table to get the scanner + * @return the number of rows + */ + public static int countMobRows(final Table table) throws IOException { + Scan scan = new Scan(); + // Do not retrieve the mob data when scanning + scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); + return HBaseTestingUtility.countRows(table, scan); + } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 8ed4fbb155..bce887a9a0 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -93,6 +93,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.mob.MobTestUtil; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -295,7 +296,7 @@ public class TestMobCompactor { int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, @@ -305,7 +306,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -322,7 +323,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); // After the compaction, the files smaller than the mob compaction merge size @@ -445,7 +446,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -462,7 +463,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After first compaction: mob rows count", regionNum - * (rowNumPerRegion - delRowNum), countMobRows(table)); + * (rowNumPerRegion - delRowNum), MobTestUtil.countMobRows(table)); assertEquals("After first compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After first compaction: family1 mob file count", regionNum, @@ -482,7 +483,7 @@ public class TestMobCompactor { admin.enableTable(tableName); assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("After restoring snapshot: family1 mob file count", regionNum * count, @@ -500,7 +501,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("After second compaction: family1 mob file count", regionNum, @@ -550,7 +551,7 @@ public class TestMobCompactor { int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, @@ -559,7 +560,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -576,7 +577,7 @@ public class TestMobCompactor { waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After compaction: family1 mob file count", regionNum, @@ -763,18 +764,6 @@ public class TestMobCompactor { assertEquals(CompactionState.NONE, state); } - /** - * Gets the number of rows in the given table. - * @param table to get the scanner - * @return the number of rows - */ - private int countMobRows(final Table table) throws IOException { - Scan scan = new Scan(); - // Do not retrieve the mob data when scanning - scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - return TEST_UTIL.countRows(table, scan); - } - /** * Gets the number of cells in the given table. * @param table to get the scanner -- 2.16.2