diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 829661c..02c9ce6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2219,13 +2219,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { for (byte[] family: families) { scan.addFamily(family); } - ResultScanner results = table.getScanner(scan); - int count = 0; - for (@SuppressWarnings("unused") Result res : results) { - count++; - } - results.close(); - return count; + return countRows(table, scan); } /** @@ -2240,6 +2234,32 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } } + public int countRows(final Region region) throws IOException { + return countRows(region, new Scan()); + } + + public int countRows(final Region region, final Scan scan) throws IOException { + InternalScanner scanner = region.getScanner(scan); + try { + return countRows(scanner); + } finally { + scanner.close(); + } + } + + public int countRows(final InternalScanner scanner) throws IOException { + // Do not retrieve the mob data when scanning + int scannedCount = 0; + List results = new ArrayList(); + boolean hasMore = true; + while (hasMore) { + hasMore = scanner.next(results); + scannedCount += results.size(); + results.clear(); + } + return scannedCount; + } + /** * Return an md5 digest of the entire contents of a table. */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index f10cce3a..50a566a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -504,7 +504,7 @@ public class TestFromClientSide { byte [] endKey = regions.get(0).getRegionInfo().getEndKey(); // Count rows with a filter that stops us before passed 'endKey'. // Should be count of rows in first region. - int endKeyCount = countRows(t, createScanWithRowFilter(endKey)); + int endKeyCount = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey)); assertTrue(endKeyCount < rowCount); // How do I know I did not got to second region? Thats tough. Can't really @@ -516,29 +516,29 @@ public class TestFromClientSide { // New test. Make it so scan goes into next region by one and then two. // Make sure count comes out right. byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)}; - int plusOneCount = countRows(t, createScanWithRowFilter(key)); + int plusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); assertEquals(endKeyCount + 1, plusOneCount); key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)}; - int plusTwoCount = countRows(t, createScanWithRowFilter(key)); + int plusTwoCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); assertEquals(endKeyCount + 2, plusTwoCount); // New test. Make it so I scan one less than endkey. key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)}; - int minusOneCount = countRows(t, createScanWithRowFilter(key)); + int minusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); assertEquals(endKeyCount - 1, minusOneCount); // For above test... study logs. Make sure we do "Finished with scanning.." // in first region and that we do not fall into the next region. key = new byte [] {'a', 'a', 'a'}; - int countBBB = countRows(t, + int countBBB = TEST_UTIL.countRows(t, createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL)); assertEquals(1, countBBB); - int countGreater = countRows(t, createScanWithRowFilter(endKey, null, + int countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, null, CompareFilter.CompareOp.GREATER_OR_EQUAL)); // Because started at start of table. assertEquals(0, countGreater); - countGreater = countRows(t, createScanWithRowFilter(endKey, endKey, + countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, endKey, CompareFilter.CompareOp.GREATER_OR_EQUAL)); assertEquals(rowCount - endKeyCount, countGreater); } @@ -602,7 +602,7 @@ public class TestFromClientSide { try (Table t = TEST_UTIL.getConnection().getTable(name)) { int rowCount = TEST_UTIL.loadTable(t, FAMILY, false); TEST_UTIL.getAdmin().flush(name); - int actualRowCount = countRows(t, new Scan().addColumn(FAMILY, FAMILY)); + int actualRowCount = TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY)); assertEquals(rowCount, actualRowCount); } assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0); @@ -635,28 +635,9 @@ public class TestFromClientSide { return s; } - /* - * @param t - * @param s - * @return Count of rows in table. - * @throws IOException - */ - private int countRows(final Table t, final Scan s) - throws IOException { - // Assert all rows in table. - ResultScanner scanner = t.getScanner(s); - int count = 0; - for (Result result: scanner) { - count++; - assertTrue(result.size() > 0); - // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow())); - } - return count; - } - private void assertRowCount(final Table t, final int expected) throws IOException { - assertEquals(expected, countRows(t, new Scan())); + assertEquals(expected, TEST_UTIL.countRows(t, new Scan())); } /* diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java index 8717c65..089872b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java @@ -95,7 +95,7 @@ public class TestServerCrashProcedure { try (Table t = this.util.getConnection().getTable(tableName)) { // Load the table with a bit of data so some logs to split and some edits in each region. this.util.loadTable(t, HBaseTestingUtility.COLUMNS[0]); - int count = countRows(t); + int count = util.countRows(t); // Run the procedure executor outside the master so we can mess with it. Need to disable // Master's running of the server crash processing. HMaster master = this.util.getHBaseCluster().getMaster(); @@ -123,15 +123,7 @@ public class TestServerCrashProcedure { // Now run through the procedure twice crashing the executor on each step... MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); // Assert all data came back. - assertEquals(count, countRows(t)); + assertEquals(count, util.countRows(t)); } } - - int countRows(final Table t) throws IOException { - int count = 0; - try (ResultScanner scanner = t.getScanner(new Scan())) { - while(scanner.next() != null) count++; - } - return count; - } } \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java index 5cbca4b..6f71548 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java @@ -132,7 +132,7 @@ public class TestMobStoreCompaction { } assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles()); assertEquals("Before compaction: mob file count", 0, countMobFiles()); - assertEquals("Before compaction: rows", compactionThreshold, countRows()); + assertEquals("Before compaction: rows", compactionThreshold, UTIL.countRows(region)); assertEquals("Before compaction: mob rows", 0, countMobRows()); region.compactStores(); @@ -140,7 +140,7 @@ public class TestMobStoreCompaction { assertEquals("After compaction: store files", 1, countStoreFiles()); assertEquals("After compaction: mob file count", 0, countMobFiles()); assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles()); - assertEquals("After compaction: rows", compactionThreshold, countRows()); + assertEquals("After compaction: rows", compactionThreshold, UTIL.countRows(region)); assertEquals("After compaction: mob rows", 0, countMobRows()); } @@ -159,7 +159,7 @@ public class TestMobStoreCompaction { } assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles()); assertEquals("Before compaction: mob file count", compactionThreshold, countMobFiles()); - assertEquals("Before compaction: rows", compactionThreshold, countRows()); + assertEquals("Before compaction: rows", compactionThreshold, UTIL.countRows(region)); assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows()); assertEquals("Before compaction: number of mob cells", compactionThreshold, countMobCellsInMetadata()); @@ -171,7 +171,7 @@ public class TestMobStoreCompaction { assertEquals("After compaction: store files", 1, countStoreFiles()); assertEquals("After compaction: mob file count", compactionThreshold, countMobFiles()); assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles()); - assertEquals("After compaction: rows", compactionThreshold, countRows()); + assertEquals("After compaction: rows", compactionThreshold, UTIL.countRows(region)); assertEquals("After compaction: mob rows", 0, countMobRows()); } @@ -200,7 +200,7 @@ public class TestMobStoreCompaction { assertTrue("Bulkload result:", result); assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles()); assertEquals("Before compaction: mob file count", 0, countMobFiles()); - assertEquals("Before compaction: rows", compactionThreshold, countRows()); + assertEquals("Before compaction: rows", compactionThreshold, UTIL.countRows(region)); assertEquals("Before compaction: mob rows", 0, countMobRows()); assertEquals("Before compaction: referenced mob file count", 0, countReferencedMobFiles()); @@ -208,7 +208,7 @@ public class TestMobStoreCompaction { assertEquals("After compaction: store files", 1, countStoreFiles()); assertEquals("After compaction: mob file count:", 1, countMobFiles()); - assertEquals("After compaction: rows", compactionThreshold, countRows()); + assertEquals("After compaction: rows", compactionThreshold, UTIL.countRows(region)); assertEquals("After compaction: mob rows", compactionThreshold, countMobRows()); assertEquals("After compaction: referenced mob file count", 1, countReferencedMobFiles()); assertEquals("After compaction: number of mob cells", compactionThreshold, @@ -230,7 +230,7 @@ public class TestMobStoreCompaction { } assertEquals("Before compaction: store files", numHfiles, countStoreFiles()); assertEquals("Before compaction: mob file count", numHfiles, countMobFiles()); - assertEquals("Before compaction: rows", numHfiles, countRows()); + assertEquals("Before compaction: rows", numHfiles, UTIL.countRows(region)); assertEquals("Before compaction: mob rows", numHfiles, countMobRows()); assertEquals("Before compaction: number of mob cells", numHfiles, countMobCellsInMetadata()); // now let's delete some cells that contain mobs @@ -351,29 +351,6 @@ public class TestMobStoreCompaction { return scannedCount; } - private int countRows() throws IOException { - Scan scan = new Scan(); - InternalScanner scanner = region.getScanner(scan); - try { - return countRows(scanner); - } finally { - scanner.close(); - } - } - - private int countRows(InternalScanner scanner) throws IOException { - // Do not retrieve the mob data when scanning - int scannedCount = 0; - List results = new ArrayList(); - boolean hasMore = true; - while (hasMore) { - hasMore = scanner.next(results); - scannedCount += results.size(); - results.clear(); - } - return scannedCount; - } - private byte[] makeDummyData(int size) { byte[] dummyData = new byte[size]; new Random().nextBytes(dummyData); @@ -448,7 +425,7 @@ public class TestMobStoreCompaction { StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null, scanners, 0L, HConstants.LATEST_TIMESTAMP); try { - size += countRows(scanner); + size += UTIL.countRows(scanner); } finally { scanner.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java index 4630522..21d04d3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java @@ -244,8 +244,8 @@ public class TestRegionMergeTransaction { final int rowCountOfRegionA = loadRegion(this.region_a, CF, true); final int rowCountOfRegionB = loadRegion(this.region_b, CF, true); assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0); - assertEquals(rowCountOfRegionA, countRows(this.region_a)); - assertEquals(rowCountOfRegionB, countRows(this.region_b)); + assertEquals(rowCountOfRegionA, TEST_UTIL.countRows(this.region_a)); + assertEquals(rowCountOfRegionB, TEST_UTIL.countRows(this.region_b)); // Start transaction. RegionMergeTransactionImpl mt = prepareOnGoodRegions(); @@ -272,7 +272,7 @@ public class TestRegionMergeTransaction { mergedRegion.getRegionInfo().getEndKey())); // Count rows. merged region are already open try { - int mergedRegionRowCount = countRows(mergedRegion); + int mergedRegionRowCount = TEST_UTIL.countRows(mergedRegion); assertEquals((rowCountOfRegionA + rowCountOfRegionB), mergedRegionRowCount); } finally { @@ -288,8 +288,8 @@ public class TestRegionMergeTransaction { final int rowCountOfRegionA = loadRegion(this.region_a, CF, true); final int rowCountOfRegionB = loadRegion(this.region_b, CF, true); assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0); - assertEquals(rowCountOfRegionA, countRows(this.region_a)); - assertEquals(rowCountOfRegionB, countRows(this.region_b)); + assertEquals(rowCountOfRegionA, TEST_UTIL.countRows(this.region_a)); + assertEquals(rowCountOfRegionB, TEST_UTIL.countRows(this.region_b)); // Start transaction. RegionMergeTransactionImpl mt = prepareOnGoodRegions(); @@ -314,9 +314,9 @@ public class TestRegionMergeTransaction { assertTrue(mt.rollback(null, null)); // Assert I can scan region_a and region_b. - int rowCountOfRegionA2 = countRows(this.region_a); + int rowCountOfRegionA2 = TEST_UTIL.countRows(this.region_a); assertEquals(rowCountOfRegionA, rowCountOfRegionA2); - int rowCountOfRegionB2 = countRows(this.region_b); + int rowCountOfRegionB2 = TEST_UTIL.countRows(this.region_b); assertEquals(rowCountOfRegionB, rowCountOfRegionB2); // Assert rollback cleaned up stuff in fs @@ -332,7 +332,7 @@ public class TestRegionMergeTransaction { // Count rows. daughters are already open // Count rows. merged region are already open try { - int mergedRegionRowCount = countRows(mergedRegion); + int mergedRegionRowCount = TEST_UTIL.countRows(mergedRegion); assertEquals((rowCountOfRegionA + rowCountOfRegionB), mergedRegionRowCount); } finally { @@ -348,8 +348,8 @@ public class TestRegionMergeTransaction { final int rowCountOfRegionA = loadRegion(this.region_a, CF, true); final int rowCountOfRegionB = loadRegion(this.region_b, CF, true); assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0); - assertEquals(rowCountOfRegionA, countRows(this.region_a)); - assertEquals(rowCountOfRegionB, countRows(this.region_b)); + assertEquals(rowCountOfRegionA, TEST_UTIL.countRows(this.region_a)); + assertEquals(rowCountOfRegionB, TEST_UTIL.countRows(this.region_b)); // Start transaction. RegionMergeTransactionImpl mt = prepareOnGoodRegions(); @@ -448,23 +448,6 @@ public class TestRegionMergeTransaction { TEST_UTIL.getConfiguration()); } - private int countRows(final HRegion r) throws IOException { - int rowcount = 0; - InternalScanner scanner = r.getScanner(new Scan()); - try { - List kvs = new ArrayList(); - boolean hasNext = true; - while (hasNext) { - hasNext = scanner.next(kvs); - if (!kvs.isEmpty()) - rowcount++; - } - } finally { - scanner.close(); - } - return rowcount; - } - /** * Load region with rows from 'aaa' to 'zzz', skip the rows which are out of * range of the region diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java index 2ade27a..80be45a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java @@ -80,10 +80,10 @@ public class TestSplitTransaction { private static final byte [] ENDROW = new byte [] {'{', '{', '{'}; private static final byte [] GOOD_SPLIT_ROW = new byte [] {'d', 'd', 'd'}; private static final byte [] CF = HConstants.CATALOG_FAMILY; - + private static boolean preRollBackCalled = false; private static boolean postRollBackCalled = false; - + @Before public void setup() throws IOException { this.fs = FileSystem.get(TEST_UTIL.getConfiguration()); TEST_UTIL.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName()); @@ -91,7 +91,7 @@ public class TestSplitTransaction { final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); FSUtils.setRootDir(walConf, this.testdir); this.wals = new WALFactory(walConf, null, this.getClass().getName()); - + this.parent = createRegion(this.testdir, this.wals); RegionCoprocessorHost host = new RegionCoprocessorHost(this.parent, null, TEST_UTIL.getConfiguration()); this.parent.setCoprocessorHost(host); @@ -113,7 +113,7 @@ public class TestSplitTransaction { @Test public void testFailAfterPONR() throws IOException, KeeperException { final int rowcount = TEST_UTIL.loadRegion(this.parent, CF); assertTrue(rowcount > 0); - int parentRowCount = countRows(this.parent); + int parentRowCount = TEST_UTIL.countRows(this.parent); assertEquals(rowcount, parentRowCount); // Start transaction. @@ -229,7 +229,7 @@ public class TestSplitTransaction { @Test public void testWholesomeSplit() throws IOException { final int rowcount = TEST_UTIL.loadRegion(this.parent, CF, true); assertTrue(rowcount > 0); - int parentRowCount = countRows(this.parent); + int parentRowCount = TEST_UTIL.countRows(this.parent); assertEquals(rowcount, parentRowCount); // Pretend region's blocks are not in the cache, used for @@ -263,7 +263,7 @@ public class TestSplitTransaction { int daughtersRowCount = 0; for (Region openRegion: daughters) { try { - int count = countRows(openRegion); + int count = TEST_UTIL.countRows(openRegion); assertTrue(count > 0 && count != rowcount); daughtersRowCount += count; } finally { @@ -279,7 +279,7 @@ public class TestSplitTransaction { public void testCountReferencesFailsSplit() throws IOException { final int rowcount = TEST_UTIL.loadRegion(this.parent, CF); assertTrue(rowcount > 0); - int parentRowCount = countRows(this.parent); + int parentRowCount = TEST_UTIL.countRows(this.parent); assertEquals(rowcount, parentRowCount); // Start transaction. @@ -307,7 +307,7 @@ public class TestSplitTransaction { @Test public void testRollback() throws IOException { final int rowcount = TEST_UTIL.loadRegion(this.parent, CF); assertTrue(rowcount > 0); - int parentRowCount = countRows(this.parent); + int parentRowCount = TEST_UTIL.countRows(this.parent); assertEquals(rowcount, parentRowCount); // Start transaction. @@ -332,7 +332,7 @@ public class TestSplitTransaction { assertTrue(spiedUponSt.rollback(null, null)); // Assert I can scan parent. - int parentRowCount2 = countRows(this.parent); + int parentRowCount2 = TEST_UTIL.countRows(this.parent); assertEquals(parentRowCount, parentRowCount2); // Assert rollback cleaned up stuff in fs @@ -347,7 +347,7 @@ public class TestSplitTransaction { int daughtersRowCount = 0; for (Region openRegion: daughters) { try { - int count = countRows(openRegion); + int count = TEST_UTIL.countRows(openRegion); assertTrue(count > 0 && count != rowcount); daughtersRowCount += count; } finally { @@ -359,7 +359,7 @@ public class TestSplitTransaction { assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread()); assertTrue("Rollback hooks should be called.", wasRollBackHookCalled()); } - + private boolean wasRollBackHookCalled(){ return (preRollBackCalled && postRollBackCalled); } @@ -371,22 +371,6 @@ public class TestSplitTransaction { private class MockedFailedDaughterCreation extends IOException {} private class MockedFailedDaughterOpen extends IOException {} - private int countRows(final Region r) throws IOException { - int rowcount = 0; - InternalScanner scanner = r.getScanner(new Scan()); - try { - List kvs = new ArrayList(); - boolean hasNext = true; - while (hasNext) { - hasNext = scanner.next(kvs); - if (!kvs.isEmpty()) rowcount++; - } - } finally { - scanner.close(); - } - return rowcount; - } - HRegion createRegion(final Path testdir, final WALFactory wals) throws IOException { // Make a region with start and end keys. Use 'aaa', to 'AAA'. The load @@ -402,14 +386,14 @@ public class TestSplitTransaction { wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()), TEST_UTIL.getConfiguration()); } - + public static class CustomObserver extends BaseRegionObserver{ @Override public void preRollBackSplit( ObserverContext ctx) throws IOException { preRollBackCalled = true; } - + @Override public void postRollBackSplit( ObserverContext ctx) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java index da51516..9fe1db6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java @@ -296,28 +296,16 @@ public class BaseTestHBaseFsck { * Counts the number of rows to verify data loss or non-dataloss. */ int countRows() throws IOException { - Scan s = new Scan(); - ResultScanner rs = tbl.getScanner(s); - int i = 0; - while(rs.next() !=null) { - i++; - } - return i; + return TEST_UTIL.countRows(tbl); } /** * Counts the number of rows to verify data loss or non-dataloss. */ int countRows(byte[] start, byte[] end) throws IOException { - Scan s = new Scan(start, end); - ResultScanner rs = tbl.getScanner(s); - int i = 0; - while (rs.next() != null) { - i++; - } - return i; + return TEST_UTIL.countRows(tbl, new Scan(start, end)); } - + /** * delete table in preparation for next test *