From cf111cd5852bcff824426b779c4639a282632c8e Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Wed, 5 Sep 2018 21:23:06 -0700 Subject: [PATCH] HBASE-21138 Close HRegion instance at the end of every test in TestHRegion --- .../hbase/regionserver/TestHRegion.java | 4735 ++++++++--------- 1 file changed, 2194 insertions(+), 2541 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index a10f48d9fd..f72126fbe0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -235,6 +235,8 @@ public class TestHRegion { @After public void tearDown() throws Exception { + // Region may have been closed, but it is still no harm if we close it again here using HTU. + HBaseTestingUtility.closeRegionAndWAL(region); EnvironmentEdgeManagerTestHelper.reset(); LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir()); TEST_UTIL.cleanupTestDir(); @@ -250,7 +252,7 @@ public class TestHRegion { */ @Test (timeout = 100000) public void testSequenceId() throws IOException { - HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES); + region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES); assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); // Weird. This returns 0 if no store files or no edits. Afraid to change it. assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); @@ -269,7 +271,6 @@ public class TestHRegion { assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); region.flush(true); long max = region.getMaxFlushedSeqId(); - region.close(); assertEquals(max, region.getMaxFlushedSeqId()); } @@ -285,7 +286,7 @@ public class TestHRegion { */ @Test (timeout=60000) public void testCloseCarryingSnapshot() throws IOException { - HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES); + region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES); Store store = region.getStore(COLUMN_FAMILY_BYTES); // Get some random bytes. byte [] value = Bytes.toBytes(name.getMethodName()); @@ -302,11 +303,8 @@ public class TestHRegion { // Close with something in memstore and something in the snapshot. Make sure all is cleared. region.close(); assertEquals(0, region.getMemstoreSize()); - HRegion.closeHRegion(region); } - - /* * This test is for verifying memstore snapshot size is correctly updated in case of rollback * See HBASE-10845 @@ -334,7 +332,7 @@ public class TestHRegion { FileSystem fs = FileSystem.get(CONF); Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); - HRegion region = initHRegion(tableName, null, null, name.getMethodName(), + region = initHRegion(tableName, null, null, name.getMethodName(), CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES); Store store = region.getStore(COLUMN_FAMILY_BYTES); @@ -356,7 +354,6 @@ public class TestHRegion { } long sz = store.getFlushableSize(); assertTrue("flushable size should be zero, but it is " + sz, sz == 0); - HRegion.closeHRegion(region); } /** @@ -367,7 +364,7 @@ public class TestHRegion { FileSystem fs = FileSystem.get(CONF); Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling"); FSHLog hLog = new FSHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF); - HRegion region = initHRegion(tableName, null, null, name.getMethodName(), + region = initHRegion(tableName, null, null, name.getMethodName(), CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES); Store store = region.getStore(COLUMN_FAMILY_BYTES); assertEquals(0, region.getMemstoreSize()); @@ -399,7 +396,6 @@ public class TestHRegion { region.flush(true); assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize()); assertEquals("flushable size should be zero", 0, store.getFlushableSize()); - HRegion.closeHRegion(region); } @Test @@ -408,7 +404,7 @@ public class TestHRegion { FileSystem fs = FileSystem.get(CONF); Path rootDir = new Path(dir + testName); FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF); - HRegion region = initHRegion(tableName, null, null, name.getMethodName(), + region = initHRegion(tableName, null, null, name.getMethodName(), CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES); Store store = region.getStore(COLUMN_FAMILY_BYTES); assertEquals(0, region.getMemstoreSize()); @@ -438,7 +434,6 @@ public class TestHRegion { assertEquals("flushable size should be incremented", expectedSize, store.getFlushableSize()); region.setCoprocessorHost(null); - HBaseTestingUtility.closeRegionAndWAL(region); } /** @@ -778,32 +773,27 @@ public class TestHRegion { public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, method, CONF, family); - try { - Path regiondir = region.getRegionFileSystem().getRegionDir(); - FileSystem fs = region.getRegionFileSystem().getFileSystem(); + Path regiondir = region.getRegionFileSystem().getRegionDir(); + FileSystem fs = region.getRegionFileSystem().getFileSystem(); - Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); - for (int i = 1000; i < 1050; i += 10) { - Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); - FSDataOutputStream dos = fs.create(recoveredEdits); - dos.writeInt(i); - dos.close(); - } - long minSeqId = 2000; - Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); + Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); + for (int i = 1000; i < 1050; i += 10) { + Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); FSDataOutputStream dos = fs.create(recoveredEdits); + dos.writeInt(i); dos.close(); + } + long minSeqId = 2000; + Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); + FSDataOutputStream dos = fs.create(recoveredEdits); + dos.close(); - Map maxSeqIdInStores = new TreeMap(Bytes.BYTES_COMPARATOR); - for (Store store : region.getStores()) { - maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId); - } - long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null); - assertEquals(minSeqId, seqId); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + Map maxSeqIdInStores = new TreeMap(Bytes.BYTES_COMPARATOR); + for (Store store : region.getStores()) { + maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId); } + long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null); + assertEquals(minSeqId, seqId); } @Test @@ -1202,57 +1192,51 @@ public class TestHRegion { getName(), walConf); this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family); - try { - int i = 0; - Put put = new Put(Bytes.toBytes(i)); - put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal - put.add(family, Bytes.toBytes(i), Bytes.toBytes(i)); - region.put(put); + int i = 0; + Put put = new Put(Bytes.toBytes(i)); + put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal + put.add(family, Bytes.toBytes(i), Bytes.toBytes(i)); + region.put(put); - // 1. Test case where START_FLUSH throws exception - wal.flushActions = new FlushAction [] {FlushAction.START_FLUSH}; + // 1. Test case where START_FLUSH throws exception + wal.flushActions = new FlushAction [] {FlushAction.START_FLUSH}; - // start cache flush will throw exception - try { - region.flush(true); - fail("This should have thrown exception"); - } catch (DroppedSnapshotException unexpected) { - // this should not be a dropped snapshot exception. Meaning that RS will not abort - throw unexpected; - } catch (IOException expected) { - // expected - } - // The WAL is hosed now. It has two edits appended. We cannot roll the log without it - // throwing a DroppedSnapshotException to force an abort. Just clean up the mess. - region.close(true); - wal.close(); - - // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception - wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH}; - wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf), - getName(), walConf); - this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW, - HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family); - region.put(put); + // start cache flush will throw exception + try { + region.flush(true); + fail("This should have thrown exception"); + } catch (DroppedSnapshotException unexpected) { + // this should not be a dropped snapshot exception. Meaning that RS will not abort + throw unexpected; + } catch (IOException expected) { + // expected + } + // The WAL is hosed now. It has two edits appended. We cannot roll the log without it + // throwing a DroppedSnapshotException to force an abort. Just clean up the mess. + region.close(true); + wal.close(); - // 3. Test case where ABORT_FLUSH will throw exception. - // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with - // DroppedSnapshotException. Below COMMMIT_FLUSH will cause flush to abort - wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH}; + // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception + wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH}; + wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf), + getName(), walConf); + this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW, + HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family); + region.put(put); - try { - region.flush(true); - fail("This should have thrown exception"); - } catch (DroppedSnapshotException expected) { - // we expect this exception, since we were able to write the snapshot, but failed to - // write the flush marker to WAL - } catch (IOException unexpected) { - throw unexpected; - } + // 3. Test case where ABORT_FLUSH will throw exception. + // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with + // DroppedSnapshotException. Below COMMMIT_FLUSH will cause flush to abort + wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH}; - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + try { + region.flush(true); + fail("This should have thrown exception"); + } catch (DroppedSnapshotException expected) { + // we expect this exception, since we were able to write the snapshot, but failed to + // write the flush marker to WAL + } catch (IOException unexpected) { + throw unexpected; } } @@ -1266,50 +1250,45 @@ public class TestHRegion { // Setting up region String method = name.getMethodName(); this.region = initHRegion(tableName, method, hc, families); + // Put data in region + final int startRow = 100; + putData(startRow, numRows, qual1, families); + putData(startRow, numRows, qual2, families); + putData(startRow, numRows, qual3, families); + final AtomicBoolean done = new AtomicBoolean(false); + final AtomicInteger gets = new AtomicInteger(0); + GetTillDoneOrException[] threads = new GetTillDoneOrException[10]; try { - // Put data in region - final int startRow = 100; - putData(startRow, numRows, qual1, families); - putData(startRow, numRows, qual2, families); - putData(startRow, numRows, qual3, families); - final AtomicBoolean done = new AtomicBoolean(false); - final AtomicInteger gets = new AtomicInteger(0); - GetTillDoneOrException[] threads = new GetTillDoneOrException[10]; - try { - // Set ten threads running concurrently getting from the region. - for (int i = 0; i < threads.length / 2; i++) { - threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); - threads[i].setDaemon(true); - threads[i].start(); - } - // Artificially make the condition by setting closing flag explicitly. - // I can't make the issue happen with a call to region.close(). - this.region.closing.set(true); - for (int i = threads.length / 2; i < threads.length; i++) { - threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); - threads[i].setDaemon(true); - threads[i].start(); - } - } finally { - if (this.region != null) { - HRegion.closeHRegion(this.region); - } + // Set ten threads running concurrently getting from the region. + for (int i = 0; i < threads.length / 2; i++) { + threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); + threads[i].setDaemon(true); + threads[i].start(); } - done.set(true); - for (GetTillDoneOrException t : threads) { - try { - t.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if (t.e != null) { - LOG.info("Exception=" + t.e); - assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException); - } + // Artificially make the condition by setting closing flag explicitly. + // I can't make the issue happen with a call to region.close(). + this.region.closing.set(true); + for (int i = threads.length / 2; i < threads.length; i++) { + threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); + threads[i].setDaemon(true); + threads[i].start(); } } finally { - HRegion.closeHRegion(this.region); - this.region = null; + if (this.region != null) { + HRegion.closeHRegion(this.region); + } + } + done.set(true); + for (GetTillDoneOrException t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (t.e != null) { + LOG.info("Exception=" + t.e); + assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException); + } } } @@ -1353,41 +1332,36 @@ public class TestHRegion { byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; this.region = initHRegion(TABLE, getName(), CONF, FAMILIES); - try { - String value = "this is the value"; - String value2 = "this is some other value"; - String keyPrefix1 = "prefix1"; - String keyPrefix2 = "prefix2"; - String keyPrefix3 = "prefix3"; - putRows(this.region, 3, value, keyPrefix1); - putRows(this.region, 3, value, keyPrefix2); - putRows(this.region, 3, value, keyPrefix3); - putRows(this.region, 3, value2, keyPrefix1); - putRows(this.region, 3, value2, keyPrefix2); - putRows(this.region, 3, value2, keyPrefix3); - System.out.println("Checking values for key: " + keyPrefix1); - assertEquals("Got back incorrect number of rows from scan", 3, - getNumberOfRows(keyPrefix1, value2, this.region)); - System.out.println("Checking values for key: " + keyPrefix2); - assertEquals("Got back incorrect number of rows from scan", 3, - getNumberOfRows(keyPrefix2, value2, this.region)); - System.out.println("Checking values for key: " + keyPrefix3); - assertEquals("Got back incorrect number of rows from scan", 3, - getNumberOfRows(keyPrefix3, value2, this.region)); - deleteColumns(this.region, value2, keyPrefix1); - deleteColumns(this.region, value2, keyPrefix2); - deleteColumns(this.region, value2, keyPrefix3); - System.out.println("Starting important checks....."); - assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0, - getNumberOfRows(keyPrefix1, value2, this.region)); - assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0, - getNumberOfRows(keyPrefix2, value2, this.region)); - assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0, - getNumberOfRows(keyPrefix3, value2, this.region)); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + String value = "this is the value"; + String value2 = "this is some other value"; + String keyPrefix1 = "prefix1"; + String keyPrefix2 = "prefix2"; + String keyPrefix3 = "prefix3"; + putRows(this.region, 3, value, keyPrefix1); + putRows(this.region, 3, value, keyPrefix2); + putRows(this.region, 3, value, keyPrefix3); + putRows(this.region, 3, value2, keyPrefix1); + putRows(this.region, 3, value2, keyPrefix2); + putRows(this.region, 3, value2, keyPrefix3); + System.out.println("Checking values for key: " + keyPrefix1); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix1, value2, this.region)); + System.out.println("Checking values for key: " + keyPrefix2); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix2, value2, this.region)); + System.out.println("Checking values for key: " + keyPrefix3); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix3, value2, this.region)); + deleteColumns(this.region, value2, keyPrefix1); + deleteColumns(this.region, value2, keyPrefix2); + deleteColumns(this.region, value2, keyPrefix3); + System.out.println("Starting important checks....."); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0, + getNumberOfRows(keyPrefix1, value2, this.region)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0, + getNumberOfRows(keyPrefix2, value2, this.region)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0, + getNumberOfRows(keyPrefix3, value2, this.region)); } @Test @@ -1403,11 +1377,8 @@ public class TestHRegion { region.append(append); } catch (IOException e) { exceptionCaught = true; - } finally { - HRegion.closeHRegion(this.region); - this.region = null; } - assertTrue(exceptionCaught == true); + assertTrue(exceptionCaught); } @Test @@ -1422,11 +1393,8 @@ public class TestHRegion { region.increment(inc); } catch (IOException e) { exceptionCaught = true; - } finally { - HRegion.closeHRegion(this.region); - this.region = null; } - assertTrue(exceptionCaught == true); + assertTrue(exceptionCaught); } private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException { @@ -1507,21 +1475,16 @@ public class TestHRegion { byte[] b = Bytes.toBytes(getName()); byte[] cf = Bytes.toBytes(COLUMN_FAMILY); this.region = initHRegion(b, getName(), CONF, cf); + Put p = new Put(b); + byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); + p.add(cfwithcolon, cfwithcolon, cfwithcolon); + boolean exception = false; try { - Put p = new Put(b); - byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); - p.add(cfwithcolon, cfwithcolon, cfwithcolon); - boolean exception = false; - try { - this.region.put(p); - } catch (NoSuchColumnFamilyException e) { - exception = true; - } - assertTrue(exception); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + this.region.put(p); + } catch (NoSuchColumnFamilyException e) { + exception = true; } + assertTrue(exception); } @Test @@ -1531,38 +1494,33 @@ public class TestHRegion { byte[] val = Bytes.toBytes("val"); this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf); MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); - try { - long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); - metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); - - LOG.info("First a batch put with all valid puts"); - final Put[] puts = new Put[10]; - for (int i = 0; i < 10; i++) { - puts[i] = new Put(Bytes.toBytes("row_" + i)); - puts[i].add(cf, qual, val); - } + long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); - OperationStatus[] codes = this.region.batchMutate(puts); - assertEquals(10, codes.length); - for (int i = 0; i < 10; i++) { - assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); - } - metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); - - LOG.info("Next a batch put with one invalid family"); - puts[5].add(Bytes.toBytes("BAD_CF"), qual, val); - codes = this.region.batchMutate(puts); - assertEquals(10, codes.length); - for (int i = 0; i < 10; i++) { - assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, - codes[i].getOperationStatusCode()); - } + LOG.info("First a batch put with all valid puts"); + final Put[] puts = new Put[10]; + for (int i = 0; i < 10; i++) { + puts[i] = new Put(Bytes.toBytes("row_" + i)); + puts[i].add(cf, qual, val); + } - metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + OperationStatus[] codes = this.region.batchMutate(puts); + assertEquals(10, codes.length); + for (int i = 0; i < 10; i++) { + assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); + } + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); + + LOG.info("Next a batch put with one invalid family"); + puts[5].add(Bytes.toBytes("BAD_CF"), qual, val); + codes = this.region.batchMutate(puts); + assertEquals(10, codes.length); + for (int i = 0; i < 10; i++) { + assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, + codes[i].getOperationStatusCode()); } + + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); } @Test @@ -1572,86 +1530,81 @@ public class TestHRegion { byte[] val = Bytes.toBytes("val"); this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf); MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); - try { - long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); - metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); - - final Put[] puts = new Put[10]; - for (int i = 0; i < 10; i++) { - puts[i] = new Put(Bytes.toBytes("row_" + i)); - puts[i].add(cf, qual, val); - } - puts[5].add(Bytes.toBytes("BAD_CF"), qual, val); + long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); - LOG.info("batchPut will have to break into four batches to avoid row locks"); - RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2")); - RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1")); - RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3")); - RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true); + final Put[] puts = new Put[10]; + for (int i = 0; i < 10; i++) { + puts[i] = new Put(Bytes.toBytes("row_" + i)); + puts[i].add(cf, qual, val); + } + puts[5].add(Bytes.toBytes("BAD_CF"), qual, val); + LOG.info("batchPut will have to break into four batches to avoid row locks"); + RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2")); + RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1")); + RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3")); + RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true); - MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); - final AtomicReference retFromThread = new AtomicReference(); - final CountDownLatch startingPuts = new CountDownLatch(1); - final CountDownLatch startingClose = new CountDownLatch(1); - TestThread putter = new TestThread(ctx) { - @Override - public void doWork() throws IOException { - startingPuts.countDown(); - retFromThread.set(region.batchMutate(puts)); - } - }; - LOG.info("...starting put thread while holding locks"); - ctx.addThread(putter); - ctx.startThreads(); - // Now attempt to close the region from another thread. Prior to HBASE-12565 - // this would cause the in-progress batchMutate operation to to fail with - // exception because it use to release and re-acquire the close-guard lock - // between batches. Caller then didn't get status indicating which writes succeeded. - // We now expect this thread to block until the batchMutate call finishes. - Thread regionCloseThread = new TestThread(ctx) { - @Override - public void doWork() { - try { - startingPuts.await(); - // Give some time for the batch mutate to get in. - // We don't want to race with the mutate - Thread.sleep(10); - startingClose.countDown(); - HBaseTestingUtility.closeRegionAndWAL(region); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); + final AtomicReference retFromThread = new AtomicReference<>(); + final CountDownLatch startingPuts = new CountDownLatch(1); + final CountDownLatch startingClose = new CountDownLatch(1); + TestThread putter = new TestThread(ctx) { + @Override + public void doWork() throws IOException { + startingPuts.countDown(); + retFromThread.set(region.batchMutate(puts)); + } + }; + LOG.info("...starting put thread while holding locks"); + ctx.addThread(putter); + ctx.startThreads(); + + // Now attempt to close the region from another thread. Prior to HBASE-12565 + // this would cause the in-progress batchMutate operation to to fail with + // exception because it use to release and re-acquire the close-guard lock + // between batches. Caller then didn't get status indicating which writes succeeded. + // We now expect this thread to block until the batchMutate call finishes. + Thread regionCloseThread = new TestThread(ctx) { + @Override + public void doWork() { + try { + startingPuts.await(); + // Give some time for the batch mutate to get in. + // We don't want to race with the mutate + Thread.sleep(10); + startingClose.countDown(); + HBaseTestingUtility.closeRegionAndWAL(region); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); } - }; - regionCloseThread.start(); + } + }; + regionCloseThread.start(); - startingClose.await(); - startingPuts.await(); - Thread.sleep(100); - LOG.info("...releasing row lock 1, which should let put thread continue"); - rowLock1.release(); - rowLock2.release(); - rowLock3.release(); - waitForCounter(source, "syncTimeNumOps", syncs + 1); + startingClose.await(); + startingPuts.await(); + Thread.sleep(100); + LOG.info("...releasing row lock 1, which should let put thread continue"); + rowLock1.release(); + rowLock2.release(); + rowLock3.release(); + waitForCounter(source, "syncTimeNumOps", syncs + 1); - LOG.info("...joining on put thread"); - ctx.stop(); - regionCloseThread.join(); + LOG.info("...joining on put thread"); + ctx.stop(); + regionCloseThread.join(); - OperationStatus[] codes = retFromThread.get(); - for (int i = 0; i < codes.length; i++) { - assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, - codes[i].getOperationStatusCode()); - } - rowLock4.release(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + OperationStatus[] codes = retFromThread.get(); + for (int i = 0; i < codes.length; i++) { + assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, + codes[i].getOperationStatusCode()); } + rowLock4.release(); } private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount) @@ -1678,29 +1631,22 @@ public class TestHRegion { CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); this.region = initHRegion(b, getName(), CONF, cf); - try { - MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); - long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); - metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); - - final Put[] puts = new Put[10]; - for (int i = 0; i < 10; i++) { - puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100); - puts[i].add(cf, qual, val); - } - - OperationStatus[] codes = this.region.batchMutate(puts); - assertEquals(10, codes.length); - for (int i = 0; i < 10; i++) { - assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); - } - metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); + MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); + long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + final Put[] puts = new Put[10]; + for (int i = 0; i < 10; i++) { + puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100); + puts[i].add(cf, qual, val); } + OperationStatus[] codes = this.region.batchMutate(puts); + assertEquals(10, codes.length); + for (int i = 0; i < 10; i++) { + assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); + } + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); } // //////////////////////////////////////////////////////////////////////////// @@ -1718,67 +1664,62 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Putting empty data in key - Put put = new Put(row1); - put.add(fam1, qf1, emptyVal); - - // checkAndPut with empty value - boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( - emptyVal), put, true); - assertTrue(res); - - // Putting data in key - put = new Put(row1); - put.add(fam1, qf1, val1); - - // checkAndPut with correct value - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), - put, true); - assertTrue(res); - - // not empty anymore - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), - put, true); - assertFalse(res); - - Delete delete = new Delete(row1); - delete.deleteColumn(fam1, qf1); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), - delete, true); - assertFalse(res); - - put = new Put(row1); - put.add(fam1, qf1, val2); - // checkAndPut with correct value - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), - put, true); - assertTrue(res); - - // checkAndDelete with correct value - delete = new Delete(row1); - delete.deleteColumn(fam1, qf1); - delete.deleteColumn(fam1, qf1); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2), - delete, true); - assertTrue(res); - - delete = new Delete(row1); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), - delete, true); - assertTrue(res); - - // checkAndPut looking for a null value - put = new Put(row1); - put.add(fam1, qf1, val1); - - res = region - .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true); - assertTrue(res); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // Putting empty data in key + Put put = new Put(row1); + put.add(fam1, qf1, emptyVal); + + // checkAndPut with empty value + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( + emptyVal), put, true); + assertTrue(res); + + // Putting data in key + put = new Put(row1); + put.add(fam1, qf1, val1); + + // checkAndPut with correct value + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), + put, true); + assertTrue(res); + + // not empty anymore + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), + put, true); + assertFalse(res); + + Delete delete = new Delete(row1); + delete.deleteColumn(fam1, qf1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), + delete, true); + assertFalse(res); + + put = new Put(row1); + put.add(fam1, qf1, val2); + // checkAndPut with correct value + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), + put, true); + assertTrue(res); + + // checkAndDelete with correct value + delete = new Delete(row1); + delete.deleteColumn(fam1, qf1); + delete.deleteColumn(fam1, qf1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2), + delete, true); + assertTrue(res); + + delete = new Delete(row1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), + delete, true); + assertTrue(res); + + // checkAndPut looking for a null value + put = new Put(row1); + put.add(fam1, qf1, val1); + + res = region + .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true); + assertTrue(res); } @Test @@ -1792,27 +1733,22 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Putting data in key - Put put = new Put(row1); - put.add(fam1, qf1, val1); - region.put(put); + // Putting data in key + Put put = new Put(row1); + put.add(fam1, qf1, val1); + region.put(put); - // checkAndPut with wrong value - boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( - val2), put, true); - assertEquals(false, res); - - // checkAndDelete with wrong value - Delete delete = new Delete(row1); - delete.deleteFamily(fam1); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2), - put, true); - assertEquals(false, res); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // checkAndPut with wrong value + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( + val2), put, true); + assertFalse(res); + + // checkAndDelete with wrong value + Delete delete = new Delete(row1); + delete.deleteFamily(fam1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2), + put, true); + assertFalse(res); } @Test @@ -1825,27 +1761,22 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Putting data in key - Put put = new Put(row1); - put.add(fam1, qf1, val1); - region.put(put); + // Putting data in key + Put put = new Put(row1); + put.add(fam1, qf1, val1); + region.put(put); - // checkAndPut with correct value - boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( - val1), put, true); - assertEquals(true, res); - - // checkAndDelete with correct value - Delete delete = new Delete(row1); - delete.deleteColumn(fam1, qf1); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), - delete, true); - assertEquals(true, res); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // checkAndPut with correct value + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( + val1), put, true); + assertTrue( res); + + // checkAndDelete with correct value + Delete delete = new Delete(row1); + delete.deleteColumn(fam1, qf1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), + delete, true); + assertTrue(res); } @Test @@ -1861,86 +1792,81 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Putting val3 in key - Put put = new Put(row1); - put.add(fam1, qf1, val3); - region.put(put); + // Putting val3 in key + Put put = new Put(row1); + put.add(fam1, qf1, val3); + region.put(put); - // Test CompareOp.LESS: original = val3, compare with val3, fail - boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, - new BinaryComparator(val3), put, true); - assertEquals(false, res); - - // Test CompareOp.LESS: original = val3, compare with val4, fail - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, - new BinaryComparator(val4), put, true); - assertEquals(false, res); - - // Test CompareOp.LESS: original = val3, compare with val2, - // succeed (now value = val2) - put = new Put(row1); - put.add(fam1, qf1, val2); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, - new BinaryComparator(val2), put, true); - assertEquals(true, res); - - // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, - new BinaryComparator(val3), put, true); - assertEquals(false, res); - - // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, - // succeed (value still = val2) - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, - new BinaryComparator(val2), put, true); - assertEquals(true, res); - - // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, - // succeed (now value = val3) - put = new Put(row1); - put.add(fam1, qf1, val3); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, - new BinaryComparator(val1), put, true); - assertEquals(true, res); - - // Test CompareOp.GREATER: original = val3, compare with val3, fail - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, - new BinaryComparator(val3), put, true); - assertEquals(false, res); - - // Test CompareOp.GREATER: original = val3, compare with val2, fail - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, - new BinaryComparator(val2), put, true); - assertEquals(false, res); - - // Test CompareOp.GREATER: original = val3, compare with val4, - // succeed (now value = val2) - put = new Put(row1); - put.add(fam1, qf1, val2); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, - new BinaryComparator(val4), put, true); - assertEquals(true, res); - - // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(val1), put, true); - assertEquals(false, res); - - // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, - // succeed (value still = val2) - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(val2), put, true); - assertEquals(true, res); - - // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(val3), put, true); - assertEquals(true, res); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // Test CompareOp.LESS: original = val3, compare with val3, fail + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, + new BinaryComparator(val3), put, true); + assertEquals(false, res); + + // Test CompareOp.LESS: original = val3, compare with val4, fail + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, + new BinaryComparator(val4), put, true); + assertEquals(false, res); + + // Test CompareOp.LESS: original = val3, compare with val2, + // succeed (now value = val2) + put = new Put(row1); + put.add(fam1, qf1, val2); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, + new BinaryComparator(val2), put, true); + assertEquals(true, res); + + // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, + new BinaryComparator(val3), put, true); + assertEquals(false, res); + + // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, + // succeed (value still = val2) + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, + new BinaryComparator(val2), put, true); + assertEquals(true, res); + + // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, + // succeed (now value = val3) + put = new Put(row1); + put.add(fam1, qf1, val3); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, + new BinaryComparator(val1), put, true); + assertEquals(true, res); + + // Test CompareOp.GREATER: original = val3, compare with val3, fail + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, + new BinaryComparator(val3), put, true); + assertEquals(false, res); + + // Test CompareOp.GREATER: original = val3, compare with val2, fail + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, + new BinaryComparator(val2), put, true); + assertEquals(false, res); + + // Test CompareOp.GREATER: original = val3, compare with val4, + // succeed (now value = val2) + put = new Put(row1); + put.add(fam1, qf1, val2); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, + new BinaryComparator(val4), put, true); + assertEquals(true, res); + + // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(val1), put, true); + assertEquals(false, res); + + // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, + // succeed (value still = val2) + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(val2), put, true); + assertEquals(true, res); + + // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(val3), put, true); + assertEquals(true, res); } @Test @@ -1957,36 +1883,31 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - // Putting data in the key to check - Put put = new Put(row1); - put.add(fam1, qf1, val1); - region.put(put); + // Putting data in the key to check + Put put = new Put(row1); + put.add(fam1, qf1, val1); + region.put(put); - // Creating put to add - long ts = System.currentTimeMillis(); - KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); - put = new Put(row1); - put.add(kv); + // Creating put to add + long ts = System.currentTimeMillis(); + KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); + put = new Put(row1); + put.add(kv); - // checkAndPut with wrong value - boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( - val1), put, true); - assertEquals(true, res); + // checkAndPut with wrong value + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( + val1), put, true); + assertEquals(true, res); - Get get = new Get(row1); - get.addColumn(fam2, qf1); - Cell[] actual = region.get(get).rawCells(); + Get get = new Get(row1); + get.addColumn(fam2, qf1); + Cell[] actual = region.get(get).rawCells(); - Cell[] expected = { kv }; + Cell[] expected = { kv }; - assertEquals(expected.length, actual.length); - for (int i = 0; i < actual.length; i++) { - assertEquals(expected[i], actual[i]); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + assertEquals(expected.length, actual.length); + for (int i = 0; i < actual.length; i++) { + assertEquals(expected[i], actual[i]); } } @@ -1994,19 +1915,14 @@ public class TestHRegion { public void testCheckAndPut_wrongRowInPut() throws IOException { TableName tableName = TableName.valueOf(name.getMethodName()); this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS); + Put put = new Put(row2); + put.add(fam1, qual1, value1); try { - Put put = new Put(row2); - put.add(fam1, qual1, value1); - try { - region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, - new BinaryComparator(value2), put, false); - fail(); - } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) { - // expected exception. - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, + new BinaryComparator(value2), put, false); + fail(); + } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) { + // expected exception. } } @@ -2028,63 +1944,58 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - // Put content - Put put = new Put(row1); - put.add(fam1, qf1, val1); - region.put(put); - Threads.sleep(2); - - put = new Put(row1); - put.add(fam1, qf1, val2); - put.add(fam2, qf1, val3); - put.add(fam2, qf2, val2); - put.add(fam2, qf3, val1); - put.add(fam1, qf3, val1); - region.put(put); + // Put content + Put put = new Put(row1); + put.add(fam1, qf1, val1); + region.put(put); + Threads.sleep(2); + + put = new Put(row1); + put.add(fam1, qf1, val2); + put.add(fam2, qf1, val3); + put.add(fam2, qf2, val2); + put.add(fam2, qf3, val1); + put.add(fam1, qf3, val1); + region.put(put); - // Multi-column delete - Delete delete = new Delete(row1); - delete.deleteColumn(fam1, qf1); - delete.deleteColumn(fam2, qf1); - delete.deleteColumn(fam1, qf3); - boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( - val2), delete, true); - assertEquals(true, res); - - Get get = new Get(row1); - get.addColumn(fam1, qf1); - get.addColumn(fam1, qf3); - get.addColumn(fam2, qf2); - Result r = region.get(get); - assertEquals(2, r.size()); - assertArrayEquals(val1, r.getValue(fam1, qf1)); - assertArrayEquals(val2, r.getValue(fam2, qf2)); - - // Family delete - delete = new Delete(row1); - delete.deleteFamily(fam2); - res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), - delete, true); - assertEquals(true, res); - - get = new Get(row1); - r = region.get(get); - assertEquals(1, r.size()); - assertArrayEquals(val1, r.getValue(fam1, qf1)); - - // Row delete - delete = new Delete(row1); - res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), - delete, true); - assertEquals(true, res); - get = new Get(row1); - r = region.get(get); - assertEquals(0, r.size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // Multi-column delete + Delete delete = new Delete(row1); + delete.deleteColumn(fam1, qf1); + delete.deleteColumn(fam2, qf1); + delete.deleteColumn(fam1, qf3); + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator( + val2), delete, true); + assertEquals(true, res); + + Get get = new Get(row1); + get.addColumn(fam1, qf1); + get.addColumn(fam1, qf3); + get.addColumn(fam2, qf2); + Result r = region.get(get); + assertEquals(2, r.size()); + assertArrayEquals(val1, r.getValue(fam1, qf1)); + assertArrayEquals(val2, r.getValue(fam2, qf2)); + + // Family delete + delete = new Delete(row1); + delete.deleteFamily(fam2); + res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), + delete, true); + assertEquals(true, res); + + get = new Get(row1); + r = region.get(get); + assertEquals(1, r.size()); + assertArrayEquals(val1, r.getValue(fam1, qf1)); + + // Row delete + delete = new Delete(row1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), + delete, true); + assertEquals(true, res); + get = new Get(row1); + r = region.get(get); + assertEquals(0, r.size()); } // //////////////////////////////////////////////////////////////////////////// @@ -2103,23 +2014,18 @@ public class TestHRegion { String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - region.put(put); + region.put(put); - // We do support deleting more than 1 'latest' version - Delete delete = new Delete(row1); - delete.deleteColumn(fam1, qual); - delete.deleteColumn(fam1, qual); - region.delete(delete); + // We do support deleting more than 1 'latest' version + Delete delete = new Delete(row1); + delete.deleteColumn(fam1, qual); + delete.deleteColumn(fam1, qual); + region.delete(delete); - Get get = new Get(row1); - get.addFamily(fam1); - Result r = region.get(get); - assertEquals(0, r.size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + Get get = new Get(row1); + get.addFamily(fam1); + Result r = region.get(get); + assertEquals(0, r.size()); } @Test @@ -2133,37 +2039,32 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3); - try { - List kvs = new ArrayList(); - kvs.add(new KeyValue(row1, fam4, null, null)); + List kvs = new ArrayList(); + kvs.add(new KeyValue(row1, fam4, null, null)); - // testing existing family - byte[] family = fam2; - try { - NavigableMap> deleteMap = new TreeMap>( - Bytes.BYTES_COMPARATOR); - deleteMap.put(family, kvs); - region.delete(deleteMap, Durability.SYNC_WAL); - } catch (Exception e) { - assertTrue("Family " + new String(family) + " does not exist", false); - } + // testing existing family + byte[] family = fam2; + try { + NavigableMap> deleteMap = new TreeMap>( + Bytes.BYTES_COMPARATOR); + deleteMap.put(family, kvs); + region.delete(deleteMap, Durability.SYNC_WAL); + } catch (Exception e) { + assertTrue("Family " + new String(family) + " does not exist", false); + } - // testing non existing family - boolean ok = false; - family = fam4; - try { - NavigableMap> deleteMap = new TreeMap>( - Bytes.BYTES_COMPARATOR); - deleteMap.put(family, kvs); - region.delete(deleteMap, Durability.SYNC_WAL); - } catch (Exception e) { - ok = true; - } - assertEquals("Family " + new String(family) + " does exist", true, ok); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // testing non existing family + boolean ok = false; + family = fam4; + try { + NavigableMap> deleteMap = new TreeMap>( + Bytes.BYTES_COMPARATOR); + deleteMap.put(family, kvs); + region.delete(deleteMap, Durability.SYNC_WAL); + } catch (Exception e) { + ok = true; } + assertTrue("Family " + new String(family) + " does exist", ok); } @Test @@ -2172,66 +2073,61 @@ public class TestHRegion { byte[][] families = { fam }; String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); + EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); - byte[] row = Bytes.toBytes("table_name"); - // column names - byte[] serverinfo = Bytes.toBytes("serverinfo"); - byte[] splitA = Bytes.toBytes("splitA"); - byte[] splitB = Bytes.toBytes("splitB"); + byte[] row = Bytes.toBytes("table_name"); + // column names + byte[] serverinfo = Bytes.toBytes("serverinfo"); + byte[] splitA = Bytes.toBytes("splitA"); + byte[] splitB = Bytes.toBytes("splitB"); - // add some data: - Put put = new Put(row); - put.add(fam, splitA, Bytes.toBytes("reference_A")); - region.put(put); + // add some data: + Put put = new Put(row); + put.add(fam, splitA, Bytes.toBytes("reference_A")); + region.put(put); - put = new Put(row); - put.add(fam, splitB, Bytes.toBytes("reference_B")); - region.put(put); + put = new Put(row); + put.add(fam, splitB, Bytes.toBytes("reference_B")); + region.put(put); - put = new Put(row); - put.add(fam, serverinfo, Bytes.toBytes("ip_address")); - region.put(put); + put = new Put(row); + put.add(fam, serverinfo, Bytes.toBytes("ip_address")); + region.put(put); - // ok now delete a split: - Delete delete = new Delete(row); - delete.deleteColumns(fam, splitA); - region.delete(delete); + // ok now delete a split: + Delete delete = new Delete(row); + delete.deleteColumns(fam, splitA); + region.delete(delete); - // assert some things: - Get get = new Get(row).addColumn(fam, serverinfo); - Result result = region.get(get); - assertEquals(1, result.size()); + // assert some things: + Get get = new Get(row).addColumn(fam, serverinfo); + Result result = region.get(get); + assertEquals(1, result.size()); - get = new Get(row).addColumn(fam, splitA); - result = region.get(get); - assertEquals(0, result.size()); + get = new Get(row).addColumn(fam, splitA); + result = region.get(get); + assertEquals(0, result.size()); - get = new Get(row).addColumn(fam, splitB); - result = region.get(get); - assertEquals(1, result.size()); + get = new Get(row).addColumn(fam, splitB); + result = region.get(get); + assertEquals(1, result.size()); - // Assert that after a delete, I can put. - put = new Put(row); - put.add(fam, splitA, Bytes.toBytes("reference_A")); - region.put(put); - get = new Get(row); - result = region.get(get); - assertEquals(3, result.size()); - - // Now delete all... then test I can add stuff back - delete = new Delete(row); - region.delete(delete); - assertEquals(0, region.get(get).size()); - - region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A"))); - result = region.get(get); - assertEquals(1, result.size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // Assert that after a delete, I can put. + put = new Put(row); + put.add(fam, splitA, Bytes.toBytes("reference_A")); + region.put(put); + get = new Get(row); + result = region.get(get); + assertEquals(3, result.size()); + + // Now delete all... then test I can add stuff back + delete = new Delete(row); + region.delete(delete); + assertEquals(0, region.get(get).size()); + + region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A"))); + result = region.get(get); + assertEquals(1, result.size()); } @Test @@ -2240,37 +2136,32 @@ public class TestHRegion { byte[][] families = { fam }; String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - byte[] row = Bytes.toBytes("table_name"); - // column names - byte[] serverinfo = Bytes.toBytes("serverinfo"); + byte[] row = Bytes.toBytes("table_name"); + // column names + byte[] serverinfo = Bytes.toBytes("serverinfo"); - // add data in the far future - Put put = new Put(row); - put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value")); - region.put(put); + // add data in the far future + Put put = new Put(row); + put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value")); + region.put(put); - // now delete something in the present - Delete delete = new Delete(row); - region.delete(delete); + // now delete something in the present + Delete delete = new Delete(row); + region.delete(delete); - // make sure we still see our data - Get get = new Get(row).addColumn(fam, serverinfo); - Result result = region.get(get); - assertEquals(1, result.size()); + // make sure we still see our data + Get get = new Get(row).addColumn(fam, serverinfo); + Result result = region.get(get); + assertEquals(1, result.size()); - // delete the future row - delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3); - region.delete(delete); + // delete the future row + delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3); + region.delete(delete); - // make sure it is gone - get = new Get(row).addColumn(fam, serverinfo); - result = region.get(get); - assertEquals(0, result.size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // make sure it is gone + get = new Get(row).addColumn(fam, serverinfo); + result = region.get(get); + assertEquals(0, result.size()); } /** @@ -2283,45 +2174,39 @@ public class TestHRegion { byte[][] families = { fam }; String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - byte[] row = Bytes.toBytes("row1"); - // column names - byte[] qual = Bytes.toBytes("qual"); + byte[] row = Bytes.toBytes("row1"); + // column names + byte[] qual = Bytes.toBytes("qual"); - // add data with LATEST_TIMESTAMP, put without WAL - Put put = new Put(row); - put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); - region.put(put); + // add data with LATEST_TIMESTAMP, put without WAL + Put put = new Put(row); + put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); + region.put(put); - // Make sure it shows up with an actual timestamp - Get get = new Get(row).addColumn(fam, qual); - Result result = region.get(get); - assertEquals(1, result.size()); - Cell kv = result.rawCells()[0]; - LOG.info("Got: " + kv); - assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", - kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); - - // Check same with WAL enabled (historically these took different - // code paths, so check both) - row = Bytes.toBytes("row2"); - put = new Put(row); - put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); - region.put(put); - - // Make sure it shows up with an actual timestamp - get = new Get(row).addColumn(fam, qual); - result = region.get(get); - assertEquals(1, result.size()); - kv = result.rawCells()[0]; - LOG.info("Got: " + kv); - assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", - kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // Make sure it shows up with an actual timestamp + Get get = new Get(row).addColumn(fam, qual); + Result result = region.get(get); + assertEquals(1, result.size()); + Cell kv = result.rawCells()[0]; + LOG.info("Got: " + kv); + assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", + kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); + + // Check same with WAL enabled (historically these took different + // code paths, so check both) + row = Bytes.toBytes("row2"); + put = new Put(row); + put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); + region.put(put); + // Make sure it shows up with an actual timestamp + get = new Get(row).addColumn(fam, qual); + result = region.get(get); + assertEquals(1, result.size()); + kv = result.rawCells()[0]; + LOG.info("Got: " + kv); + assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", + kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); } /** @@ -2340,22 +2225,17 @@ public class TestHRegion { this.region = initHRegion(tableName, method, CONF, families); boolean caughtExcep = false; try { - try { - // no TS specified == use latest. should not error - region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value"))); - // TS out of range. should error - region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000, - Bytes.toBytes("value"))); - fail("Expected IOE for TS out of configured timerange"); - } catch (FailedSanityCheckException ioe) { - LOG.debug("Received expected exception", ioe); - caughtExcep = true; - } - assertTrue("Should catch FailedSanityCheckException", caughtExcep); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // no TS specified == use latest. should not error + region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value"))); + // TS out of range. should error + region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000, + Bytes.toBytes("value"))); + fail("Expected IOE for TS out of configured timerange"); + } catch (FailedSanityCheckException ioe) { + LOG.debug("Received expected exception", ioe); + caughtExcep = true; } + assertTrue("Should catch FailedSanityCheckException", caughtExcep); } @Test @@ -2363,41 +2243,36 @@ public class TestHRegion { byte[] fam1 = Bytes.toBytes("columnA"); byte[] fam2 = Bytes.toBytes("columnB"); this.region = initHRegion(tableName, getName(), CONF, fam1, fam2); - try { - byte[] rowA = Bytes.toBytes("rowA"); - byte[] rowB = Bytes.toBytes("rowB"); + byte[] rowA = Bytes.toBytes("rowA"); + byte[] rowB = Bytes.toBytes("rowB"); - byte[] value = Bytes.toBytes("value"); + byte[] value = Bytes.toBytes("value"); - Delete delete = new Delete(rowA); - delete.deleteFamily(fam1); + Delete delete = new Delete(rowA); + delete.deleteFamily(fam1); - region.delete(delete); + region.delete(delete); - // now create data. - Put put = new Put(rowA); - put.add(fam2, null, value); - region.put(put); + // now create data. + Put put = new Put(rowA); + put.add(fam2, null, value); + region.put(put); - put = new Put(rowB); - put.add(fam1, null, value); - put.add(fam2, null, value); - region.put(put); + put = new Put(rowB); + put.add(fam1, null, value); + put.add(fam2, null, value); + region.put(put); - Scan scan = new Scan(); - scan.addFamily(fam1).addFamily(fam2); - InternalScanner s = region.getScanner(scan); - List results = new ArrayList(); - s.next(results); - assertTrue(CellUtil.matchingRow(results.get(0), rowA)); + Scan scan = new Scan(); + scan.addFamily(fam1).addFamily(fam2); + InternalScanner s = region.getScanner(scan); + List results = new ArrayList(); + s.next(results); + assertTrue(CellUtil.matchingRow(results.get(0), rowA)); - results.clear(); - s.next(results); - assertTrue(CellUtil.matchingRow(results.get(0), rowB)); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + results.clear(); + s.next(results); + assertTrue(CellUtil.matchingRow(results.get(0), rowB)); } @Test @@ -2405,7 +2280,7 @@ public class TestHRegion { FileSystem fs = FileSystem.get(CONF); Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL"); FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF); - HRegion region = initHRegion(tableName, null, null, name.getMethodName(), + region = initHRegion(tableName, null, null, name.getMethodName(), CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES); Cell originalCell = CellUtil.createCell(row, COLUMN_FAMILY_BYTES, qual1, @@ -2488,46 +2363,41 @@ public class TestHRegion { public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException { TableName tableName = TableName.valueOf(name.getMethodName()); this.region = initHRegion(tableName, getName(), CONF, fam1); - try { - EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); - Put put = new Put(row); - put.add(fam1, qual1, value1); - region.put(put); + EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); + Put put = new Put(row); + put.add(fam1, qual1, value1); + region.put(put); - // now delete the value: - region.delete(delete); + // now delete the value: + region.delete(delete); - // ok put data: - put = new Put(row); - put.add(fam1, qual1, value2); - region.put(put); + // ok put data: + put = new Put(row); + put.add(fam1, qual1, value2); + region.put(put); - // ok get: - Get get = new Get(row); - get.addColumn(fam1, qual1); - - Result r = region.get(get); - assertEquals(1, r.size()); - assertArrayEquals(value2, r.getValue(fam1, qual1)); - - // next: - Scan scan = new Scan(row); - scan.addColumn(fam1, qual1); - InternalScanner s = region.getScanner(scan); - - List results = new ArrayList(); - assertEquals(false, s.next(results)); - assertEquals(1, results.size()); - Cell kv = results.get(0); - - assertArrayEquals(value2, CellUtil.cloneValue(kv)); - assertArrayEquals(fam1, CellUtil.cloneFamily(kv)); - assertArrayEquals(qual1, CellUtil.cloneQualifier(kv)); - assertArrayEquals(row, CellUtil.cloneRow(kv)); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // ok get: + Get get = new Get(row); + get.addColumn(fam1, qual1); + + Result r = region.get(get); + assertEquals(1, r.size()); + assertArrayEquals(value2, r.getValue(fam1, qual1)); + + // next: + Scan scan = new Scan(row); + scan.addColumn(fam1, qual1); + InternalScanner s = region.getScanner(scan); + + List results = new ArrayList(); + assertEquals(false, s.next(results)); + assertEquals(1, results.size()); + Cell kv = results.get(0); + + assertArrayEquals(value2, CellUtil.cloneValue(kv)); + assertArrayEquals(fam1, CellUtil.cloneFamily(kv)); + assertArrayEquals(qual1, CellUtil.cloneQualifier(kv)); + assertArrayEquals(row, CellUtil.cloneRow(kv)); } @Test @@ -2541,32 +2411,27 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Building checkerList - List kvs = new ArrayList(); - kvs.add(new KeyValue(row1, fam1, col1, null)); - kvs.add(new KeyValue(row1, fam1, col2, null)); - kvs.add(new KeyValue(row1, fam1, col3, null)); - - NavigableMap> deleteMap = new TreeMap>( - Bytes.BYTES_COMPARATOR); - deleteMap.put(fam1, kvs); - region.delete(deleteMap, Durability.SYNC_WAL); - - // extract the key values out the memstore: - // This is kinda hacky, but better than nothing... - long now = System.currentTimeMillis(); - DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore; - Cell firstCell = memstore.activeSection.getCellSkipListSet().first(); - assertTrue(firstCell.getTimestamp() <= now); - now = firstCell.getTimestamp(); - for (Cell cell : memstore.activeSection.getCellSkipListSet()) { - assertTrue(cell.getTimestamp() <= now); - now = cell.getTimestamp(); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Building checkerList + List kvs = new ArrayList(); + kvs.add(new KeyValue(row1, fam1, col1, null)); + kvs.add(new KeyValue(row1, fam1, col2, null)); + kvs.add(new KeyValue(row1, fam1, col3, null)); + + NavigableMap> deleteMap = new TreeMap>( + Bytes.BYTES_COMPARATOR); + deleteMap.put(fam1, kvs); + region.delete(deleteMap, Durability.SYNC_WAL); + + // extract the key values out the memstore: + // This is kinda hacky, but better than nothing... + long now = System.currentTimeMillis(); + DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore; + Cell firstCell = memstore.activeSection.getCellSkipListSet().first(); + assertTrue(firstCell.getTimestamp() <= now); + now = firstCell.getTimestamp(); + for (Cell cell : memstore.activeSection.getCellSkipListSet()) { + assertTrue(cell.getTimestamp() <= now); + now = cell.getTimestamp(); } } @@ -2583,21 +2448,15 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - Get get = new Get(row1); - get.addColumn(fam2, col1); + Get get = new Get(row1); + get.addColumn(fam2, col1); - // Test - try { - region.get(get); - } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { - assertFalse(false); - return; - } - assertFalse(true); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Test + try { + region.get(get); + fail("Expecting DoNotRetryIOException in get but did not get any"); + } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { + LOG.info("Got expected DoNotRetryIOException successfully"); } } @@ -2614,43 +2473,38 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Add to memstore - Put put = new Put(row1); - put.add(fam1, col1, null); - put.add(fam1, col2, null); - put.add(fam1, col3, null); - put.add(fam1, col4, null); - put.add(fam1, col5, null); - region.put(put); + // Add to memstore + Put put = new Put(row1); + put.add(fam1, col1, null); + put.add(fam1, col2, null); + put.add(fam1, col3, null); + put.add(fam1, col4, null); + put.add(fam1, col5, null); + region.put(put); - Get get = new Get(row1); - get.addColumn(fam1, col2); - get.addColumn(fam1, col4); - // Expected result - KeyValue kv1 = new KeyValue(row1, fam1, col2); - KeyValue kv2 = new KeyValue(row1, fam1, col4); - KeyValue[] expected = { kv1, kv2 }; - - // Test - Result res = region.get(get); - assertEquals(expected.length, res.size()); - for (int i = 0; i < res.size(); i++) { - assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i])); - assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i])); - assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i])); - } + Get get = new Get(row1); + get.addColumn(fam1, col2); + get.addColumn(fam1, col4); + // Expected result + KeyValue kv1 = new KeyValue(row1, fam1, col2); + KeyValue kv2 = new KeyValue(row1, fam1, col4); + KeyValue[] expected = { kv1, kv2 }; - // Test using a filter on a Get - Get g = new Get(row1); - final int count = 2; - g.setFilter(new ColumnCountGetFilter(count)); - res = region.get(g); - assertEquals(count, res.size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Test + Result res = region.get(get); + assertEquals(expected.length, res.size()); + for (int i = 0; i < res.size(); i++) { + assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i])); + assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i])); + assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i])); } + + // Test using a filter on a Get + Get g = new Get(row1); + final int count = 2; + g.setFilter(new ColumnCountGetFilter(count)); + res = region.get(g); + assertEquals(count, res.size()); } @Test @@ -2660,16 +2514,11 @@ public class TestHRegion { String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam); - try { - Get get = new Get(row); - get.addFamily(fam); - Result r = region.get(get); + Get get = new Get(row); + get.addFamily(fam); + Result r = region.get(get); - assertTrue(r.isEmpty()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + assertTrue(r.isEmpty()); } // //////////////////////////////////////////////////////////////////////////// @@ -2682,51 +2531,46 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, hc, families); + LOG.info("" + HBaseTestCase.addContent(region, fam3)); + region.flush(true); + region.compactStores(); + region.waitForFlushesAndCompactions(); + for(Store s:region.getStores()) { + s.closeAndArchiveCompactedFiles(); + } + byte[] splitRow = region.checkSplit(); + assertNotNull(splitRow); + LOG.info("SplitRow: " + Bytes.toString(splitRow)); + HRegion[] subregions = splitRegion(region, splitRow); try { - LOG.info("" + HBaseTestCase.addContent(region, fam3)); - region.flush(true); - region.compactStores(); - region.waitForFlushesAndCompactions(); - for(Store s:region.getStores()) { - s.closeAndArchiveCompactedFiles(); - } - byte[] splitRow = region.checkSplit(); - assertNotNull(splitRow); - LOG.info("SplitRow: " + Bytes.toString(splitRow)); - HRegion[] subregions = splitRegion(region, splitRow); - try { - // Need to open the regions. - for (int i = 0; i < subregions.length; i++) { - HRegion.openHRegion(subregions[i], null); - subregions[i].compactStores(); - subregions[i].waitForFlushesAndCompactions(); - for(Store s:subregions[i].getStores()) { - s.closeAndArchiveCompactedFiles(); - } - } - Path oldRegionPath = region.getRegionFileSystem().getRegionDir(); - Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir(); - Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir(); - long startTime = System.currentTimeMillis(); - region = HRegion.mergeAdjacent(subregions[0], subregions[1]); - LOG.info("Merge regions elapsed time: " - + ((System.currentTimeMillis() - startTime) / 1000.0)); - FILESYSTEM.delete(oldRegion1, true); - FILESYSTEM.delete(oldRegion2, true); - FILESYSTEM.delete(oldRegionPath, true); - LOG.info("splitAndMerge completed."); - } finally { - for (int i = 0; i < subregions.length; i++) { - try { - HRegion.closeHRegion(subregions[i]); - } catch (IOException e) { - // Ignore. - } + // Need to open the regions. + for (int i = 0; i < subregions.length; i++) { + HRegion.openHRegion(subregions[i], null); + subregions[i].compactStores(); + subregions[i].waitForFlushesAndCompactions(); + for(Store s:subregions[i].getStores()) { + s.closeAndArchiveCompactedFiles(); } } + Path oldRegionPath = region.getRegionFileSystem().getRegionDir(); + Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir(); + Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir(); + long startTime = System.currentTimeMillis(); + region = HRegion.mergeAdjacent(subregions[0], subregions[1]); + LOG.info("Merge regions elapsed time: " + + ((System.currentTimeMillis() - startTime) / 1000.0)); + FILESYSTEM.delete(oldRegion1, true); + FILESYSTEM.delete(oldRegion2, true); + FILESYSTEM.delete(oldRegionPath, true); + LOG.info("splitAndMerge completed."); } finally { - HRegion.closeHRegion(this.region); - this.region = null; + for (int i = 0; i < subregions.length; i++) { + try { + HRegion.closeHRegion(subregions[i]); + } catch (IOException e) { + // Ignore. + } + } } } @@ -2784,18 +2628,13 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); + Scan scan = new Scan(); + scan.addFamily(fam1); + scan.addFamily(fam2); try { - Scan scan = new Scan(); - scan.addFamily(fam1); - scan.addFamily(fam2); - try { - region.getScanner(scan); - } catch (Exception e) { - assertTrue("Families could not be found in Region", false); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + region.getScanner(scan); + } catch (Exception e) { + assertTrue("Families could not be found in Region", false); } } @@ -2809,20 +2648,15 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); + Scan scan = new Scan(); + scan.addFamily(fam2); + boolean ok = false; try { - Scan scan = new Scan(); - scan.addFamily(fam2); - boolean ok = false; - try { - region.getScanner(scan); - } catch (Exception e) { - ok = true; - } - assertTrue("Families could not be found in Region", ok); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + region.getScanner(scan); + } catch (Exception e) { + ok = true; } + assertTrue("Families could not be found in Region", ok); } @Test @@ -2838,35 +2672,29 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { + // Putting data in Region + Put put = new Put(row1); + put.add(fam1, null, null); + put.add(fam2, null, null); + put.add(fam3, null, null); + put.add(fam4, null, null); + region.put(put); - // Putting data in Region - Put put = new Put(row1); - put.add(fam1, null, null); - put.add(fam2, null, null); - put.add(fam3, null, null); - put.add(fam4, null, null); - region.put(put); + Scan scan = null; + HRegion.RegionScannerImpl is = null; - Scan scan = null; - HRegion.RegionScannerImpl is = null; - - // Testing to see how many scanners that is produced by getScanner, - // starting - // with known number, 2 - current = 1 - scan = new Scan(); - scan.addFamily(fam2); - scan.addFamily(fam4); - is = (RegionScannerImpl) region.getScanner(scan); - assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size()); - - scan = new Scan(); - is = (RegionScannerImpl) region.getScanner(scan); - assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // Testing to see how many scanners that is produced by getScanner, + // starting + // with known number, 2 - current = 1 + scan = new Scan(); + scan.addFamily(fam2); + scan.addFamily(fam4); + is = (RegionScannerImpl) region.getScanner(scan); + assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size()); + + scan = new Scan(); + is = (RegionScannerImpl) region.getScanner(scan); + assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size()); } /** @@ -2889,20 +2717,15 @@ public class TestHRegion { e.printStackTrace(); fail("Got IOException during initHRegion, " + e.getMessage()); } + region.closed.set(true); try { - region.closed.set(true); - try { - region.getScanner(new Scan()); - fail("Expected to get an exception during getScanner on a region that is closed"); - } catch (NotServingRegionException e) { - // this is the correct exception that is expected - } catch (IOException e) { - fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: " - + e.getMessage()); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + region.getScanner(new Scan()); + fail("Expected to get an exception during getScanner on a region that is closed"); + } catch (NotServingRegionException e) { + // this is the correct exception that is expected + } catch (IOException e) { + fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: " + + e.getMessage()); } } @@ -2921,54 +2744,49 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - // Putting data in Region - Put put = null; - put = new Put(row1); - put.add(fam1, (byte[]) null, ts, null); - put.add(fam2, (byte[]) null, ts, null); - put.add(fam3, (byte[]) null, ts, null); - put.add(fam4, (byte[]) null, ts, null); - region.put(put); + // Putting data in Region + Put put = null; + put = new Put(row1); + put.add(fam1, (byte[]) null, ts, null); + put.add(fam2, (byte[]) null, ts, null); + put.add(fam3, (byte[]) null, ts, null); + put.add(fam4, (byte[]) null, ts, null); + region.put(put); - put = new Put(row2); - put.add(fam1, (byte[]) null, ts, null); - put.add(fam2, (byte[]) null, ts, null); - put.add(fam3, (byte[]) null, ts, null); - put.add(fam4, (byte[]) null, ts, null); - region.put(put); + put = new Put(row2); + put.add(fam1, (byte[]) null, ts, null); + put.add(fam2, (byte[]) null, ts, null); + put.add(fam3, (byte[]) null, ts, null); + put.add(fam4, (byte[]) null, ts, null); + region.put(put); - Scan scan = new Scan(); - scan.addFamily(fam2); - scan.addFamily(fam4); - InternalScanner is = region.getScanner(scan); + Scan scan = new Scan(); + scan.addFamily(fam2); + scan.addFamily(fam4); + InternalScanner is = region.getScanner(scan); - List res = null; + List res = null; - // Result 1 - List expected1 = new ArrayList(); - expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); - expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); + // Result 1 + List expected1 = new ArrayList(); + expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); + expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); - res = new ArrayList(); - is.next(res); - for (int i = 0; i < res.size(); i++) { - assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i))); - } + res = new ArrayList(); + is.next(res); + for (int i = 0; i < res.size(); i++) { + assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i))); + } - // Result 2 - List expected2 = new ArrayList(); - expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); - expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); + // Result 2 + List expected2 = new ArrayList(); + expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); + expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); - res = new ArrayList(); - is.next(res); - for (int i = 0; i < res.size(); i++) { - assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i))); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + res = new ArrayList(); + is.next(res); + for (int i = 0; i < res.size(); i++) { + assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i))); } } @@ -2987,47 +2805,42 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - // Putting data in Region - Put put = null; - KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); - KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - - KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); - KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); - KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - - put = new Put(row1); - put.add(kv13); - put.add(kv12); - put.add(kv11); - put.add(kv23); - put.add(kv22); - put.add(kv21); - region.put(put); + // Putting data in Region + Put put = null; + KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); + KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); + + KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); + KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); + KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); + + put = new Put(row1); + put.add(kv13); + put.add(kv12); + put.add(kv11); + put.add(kv23); + put.add(kv22); + put.add(kv21); + region.put(put); - // Expected - List expected = new ArrayList(); - expected.add(kv13); - expected.add(kv12); + // Expected + List expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); - Scan scan = new Scan(row1); - scan.addColumn(fam1, qf1); - scan.setMaxVersions(MAX_VERSIONS); - List actual = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); + Scan scan = new Scan(row1); + scan.addColumn(fam1, qf1); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); - boolean hasNext = scanner.next(actual); - assertEquals(false, hasNext); + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); - // Verify result - for (int i = 0; i < expected.size(); i++) { - assertEquals(expected.get(i), actual.get(i)); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Verify result + for (int i = 0; i < expected.size(); i++) { + assertEquals(expected.get(i), actual.get(i)); } } @@ -3046,51 +2859,46 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - // Putting data in Region - Put put = null; - KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); - KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - - KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); - KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); - KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - - put = new Put(row1); - put.add(kv13); - put.add(kv12); - put.add(kv11); - put.add(kv23); - put.add(kv22); - put.add(kv21); - region.put(put); - region.flush(true); + // Putting data in Region + Put put = null; + KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); + KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); + + KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); + KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); + KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); + + put = new Put(row1); + put.add(kv13); + put.add(kv12); + put.add(kv11); + put.add(kv23); + put.add(kv22); + put.add(kv21); + region.put(put); + region.flush(true); - // Expected - List expected = new ArrayList(); - expected.add(kv13); - expected.add(kv12); - expected.add(kv23); - expected.add(kv22); - - Scan scan = new Scan(row1); - scan.addColumn(fam1, qf1); - scan.addColumn(fam1, qf2); - scan.setMaxVersions(MAX_VERSIONS); - List actual = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); - - boolean hasNext = scanner.next(actual); - assertEquals(false, hasNext); - - // Verify result - for (int i = 0; i < expected.size(); i++) { - assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Expected + List expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addColumn(fam1, qf1); + scan.addColumn(fam1, qf2); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + // Verify result + for (int i = 0; i < expected.size(); i++) { + assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } } @@ -3110,69 +2918,64 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - // Putting data in Region - KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); - KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); - KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - - KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); - KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); - KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); - KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - - Put put = null; - put = new Put(row1); - put.add(kv14); - put.add(kv24); - region.put(put); - region.flush(true); + // Putting data in Region + KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); + KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); + KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - put = new Put(row1); - put.add(kv23); - put.add(kv13); - region.put(put); - region.flush(true); + KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); + KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); + KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); + KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - put = new Put(row1); - put.add(kv22); - put.add(kv12); - region.put(put); - region.flush(true); + Put put = null; + put = new Put(row1); + put.add(kv14); + put.add(kv24); + region.put(put); + region.flush(true); - put = new Put(row1); - put.add(kv21); - put.add(kv11); - region.put(put); + put = new Put(row1); + put.add(kv23); + put.add(kv13); + region.put(put); + region.flush(true); - // Expected - List expected = new ArrayList(); - expected.add(kv14); - expected.add(kv13); - expected.add(kv12); - expected.add(kv24); - expected.add(kv23); - expected.add(kv22); - - Scan scan = new Scan(row1); - scan.addColumn(fam1, qf1); - scan.addColumn(fam1, qf2); - int versions = 3; - scan.setMaxVersions(versions); - List actual = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); - - boolean hasNext = scanner.next(actual); - assertEquals(false, hasNext); - - // Verify result - for (int i = 0; i < expected.size(); i++) { - assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + put = new Put(row1); + put.add(kv22); + put.add(kv12); + region.put(put); + region.flush(true); + + put = new Put(row1); + put.add(kv21); + put.add(kv11); + region.put(put); + + // Expected + List expected = new ArrayList(); + expected.add(kv14); + expected.add(kv13); + expected.add(kv12); + expected.add(kv24); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addColumn(fam1, qf1); + scan.addColumn(fam1, qf2); + int versions = 3; + scan.setMaxVersions(versions); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + // Verify result + for (int i = 0; i < expected.size(); i++) { + assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } } @@ -3191,49 +2994,44 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, families); - try { - // Putting data in Region - Put put = null; - KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); - KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - - KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); - KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); - KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - - put = new Put(row1); - put.add(kv13); - put.add(kv12); - put.add(kv11); - put.add(kv23); - put.add(kv22); - put.add(kv21); - region.put(put); + // Putting data in Region + Put put = null; + KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); + KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); + + KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); + KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); + KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); + + put = new Put(row1); + put.add(kv13); + put.add(kv12); + put.add(kv11); + put.add(kv23); + put.add(kv22); + put.add(kv21); + region.put(put); - // Expected - List expected = new ArrayList(); - expected.add(kv13); - expected.add(kv12); - expected.add(kv23); - expected.add(kv22); - - Scan scan = new Scan(row1); - scan.addFamily(fam1); - scan.setMaxVersions(MAX_VERSIONS); - List actual = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); - - boolean hasNext = scanner.next(actual); - assertEquals(false, hasNext); - - // Verify result - for (int i = 0; i < expected.size(); i++) { - assertEquals(expected.get(i), actual.get(i)); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Expected + List expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addFamily(fam1); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + // Verify result + for (int i = 0; i < expected.size(); i++) { + assertEquals(expected.get(i), actual.get(i)); } } @@ -3251,50 +3049,45 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Putting data in Region - Put put = null; - KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); - KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - - KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); - KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); - KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - - put = new Put(row1); - put.add(kv13); - put.add(kv12); - put.add(kv11); - put.add(kv23); - put.add(kv22); - put.add(kv21); - region.put(put); - region.flush(true); + // Putting data in Region + Put put = null; + KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); + KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); + + KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); + KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); + KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); + + put = new Put(row1); + put.add(kv13); + put.add(kv12); + put.add(kv11); + put.add(kv23); + put.add(kv22); + put.add(kv21); + region.put(put); + region.flush(true); - // Expected - List expected = new ArrayList(); - expected.add(kv13); - expected.add(kv12); - expected.add(kv23); - expected.add(kv22); - - Scan scan = new Scan(row1); - scan.addFamily(fam1); - scan.setMaxVersions(MAX_VERSIONS); - List actual = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); - - boolean hasNext = scanner.next(actual); - assertEquals(false, hasNext); - - // Verify result - for (int i = 0; i < expected.size(); i++) { - assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Expected + List expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addFamily(fam1); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + // Verify result + for (int i = 0; i < expected.size(); i++) { + assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } } @@ -3302,48 +3095,43 @@ public class TestHRegion { public void testScanner_StopRow1542() throws IOException { byte[] family = Bytes.toBytes("testFamily"); this.region = initHRegion(tableName, getName(), CONF, family); - try { - byte[] row1 = Bytes.toBytes("row111"); - byte[] row2 = Bytes.toBytes("row222"); - byte[] row3 = Bytes.toBytes("row333"); - byte[] row4 = Bytes.toBytes("row444"); - byte[] row5 = Bytes.toBytes("row555"); + byte[] row1 = Bytes.toBytes("row111"); + byte[] row2 = Bytes.toBytes("row222"); + byte[] row3 = Bytes.toBytes("row333"); + byte[] row4 = Bytes.toBytes("row444"); + byte[] row5 = Bytes.toBytes("row555"); - byte[] col1 = Bytes.toBytes("Pub111"); - byte[] col2 = Bytes.toBytes("Pub222"); + byte[] col1 = Bytes.toBytes("Pub111"); + byte[] col2 = Bytes.toBytes("Pub222"); - Put put = new Put(row1); - put.add(family, col1, Bytes.toBytes(10L)); - region.put(put); + Put put = new Put(row1); + put.add(family, col1, Bytes.toBytes(10L)); + region.put(put); - put = new Put(row2); - put.add(family, col1, Bytes.toBytes(15L)); - region.put(put); + put = new Put(row2); + put.add(family, col1, Bytes.toBytes(15L)); + region.put(put); - put = new Put(row3); - put.add(family, col2, Bytes.toBytes(20L)); - region.put(put); + put = new Put(row3); + put.add(family, col2, Bytes.toBytes(20L)); + region.put(put); - put = new Put(row4); - put.add(family, col2, Bytes.toBytes(30L)); - region.put(put); + put = new Put(row4); + put.add(family, col2, Bytes.toBytes(30L)); + region.put(put); - put = new Put(row5); - put.add(family, col1, Bytes.toBytes(40L)); - region.put(put); + put = new Put(row5); + put.add(family, col1, Bytes.toBytes(40L)); + region.put(put); - Scan scan = new Scan(row3, row4); - scan.setMaxVersions(); - scan.addColumn(family, col1); - InternalScanner s = region.getScanner(scan); + Scan scan = new Scan(row3, row4); + scan.setMaxVersions(); + scan.addColumn(family, col1); + InternalScanner s = region.getScanner(scan); - List results = new ArrayList(); - assertEquals(false, s.next(results)); - assertEquals(0, results.size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + List results = new ArrayList(); + assertEquals(false, s.next(results)); + assertEquals(0, results.size()); } @Test @@ -3361,67 +3149,62 @@ public class TestHRegion { // Setting up region String method = this.getName(); this.region = initHRegion(tableName, method, CONF, fam1); - try { - // Putting data in Region - KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); - KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); - KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - - KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); - KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); - KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); - KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - - Put put = null; - put = new Put(row1); - put.add(kv14); - put.add(kv24); - region.put(put); - region.flush(true); + // Putting data in Region + KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); + KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); + KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); - put = new Put(row1); - put.add(kv23); - put.add(kv13); - region.put(put); - region.flush(true); + KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); + KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); + KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); + KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); - put = new Put(row1); - put.add(kv22); - put.add(kv12); - region.put(put); - region.flush(true); + Put put = null; + put = new Put(row1); + put.add(kv14); + put.add(kv24); + region.put(put); + region.flush(true); - put = new Put(row1); - put.add(kv21); - put.add(kv11); - region.put(put); + put = new Put(row1); + put.add(kv23); + put.add(kv13); + region.put(put); + region.flush(true); - // Expected - List expected = new ArrayList(); - expected.add(kv14); - expected.add(kv13); - expected.add(kv12); - expected.add(kv24); - expected.add(kv23); - expected.add(kv22); - - Scan scan = new Scan(row1); - int versions = 3; - scan.setMaxVersions(versions); - List actual = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); - - boolean hasNext = scanner.next(actual); - assertEquals(false, hasNext); - - // Verify result - for (int i = 0; i < expected.size(); i++) { - assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + put = new Put(row1); + put.add(kv22); + put.add(kv12); + region.put(put); + region.flush(true); + + put = new Put(row1); + put.add(kv21); + put.add(kv11); + region.put(put); + + // Expected + List expected = new ArrayList(); + expected.add(kv14); + expected.add(kv13); + expected.add(kv12); + expected.add(kv24); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + int versions = 3; + scan.setMaxVersions(versions); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + // Verify result + for (int i = 0; i < expected.size(); i++) { + assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } } @@ -3437,61 +3220,56 @@ public class TestHRegion { byte[] cf_joined = Bytes.toBytes("joined"); byte[] cf_alpha = Bytes.toBytes("alpha"); this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha); - try { - byte[] row1 = Bytes.toBytes("row1"); - byte[] row2 = Bytes.toBytes("row2"); - byte[] row3 = Bytes.toBytes("row3"); + byte[] row1 = Bytes.toBytes("row1"); + byte[] row2 = Bytes.toBytes("row2"); + byte[] row3 = Bytes.toBytes("row3"); - byte[] col_normal = Bytes.toBytes("d"); - byte[] col_alpha = Bytes.toBytes("a"); + byte[] col_normal = Bytes.toBytes("d"); + byte[] col_alpha = Bytes.toBytes("a"); - byte[] filtered_val = Bytes.toBytes(3); + byte[] filtered_val = Bytes.toBytes(3); - Put put = new Put(row1); - put.add(cf_essential, col_normal, Bytes.toBytes(1)); - put.add(cf_joined, col_alpha, Bytes.toBytes(1)); - region.put(put); + Put put = new Put(row1); + put.add(cf_essential, col_normal, Bytes.toBytes(1)); + put.add(cf_joined, col_alpha, Bytes.toBytes(1)); + region.put(put); - put = new Put(row2); - put.add(cf_essential, col_alpha, Bytes.toBytes(2)); - put.add(cf_joined, col_normal, Bytes.toBytes(2)); - put.add(cf_alpha, col_alpha, Bytes.toBytes(2)); - region.put(put); + put = new Put(row2); + put.add(cf_essential, col_alpha, Bytes.toBytes(2)); + put.add(cf_joined, col_normal, Bytes.toBytes(2)); + put.add(cf_alpha, col_alpha, Bytes.toBytes(2)); + region.put(put); - put = new Put(row3); - put.add(cf_essential, col_normal, filtered_val); - put.add(cf_joined, col_normal, filtered_val); - region.put(put); + put = new Put(row3); + put.add(cf_essential, col_normal, filtered_val); + put.add(cf_joined, col_normal, filtered_val); + region.put(put); - // Check two things: - // 1. result list contains expected values - // 2. result list is sorted properly + // Check two things: + // 1. result list contains expected values + // 2. result list is sorted properly - Scan scan = new Scan(); - Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, - CompareOp.NOT_EQUAL, filtered_val); - scan.setFilter(filter); - scan.setLoadColumnFamiliesOnDemand(true); - InternalScanner s = region.getScanner(scan); - - List results = new ArrayList(); - assertTrue(s.next(results)); - assertEquals(results.size(), 1); - results.clear(); + Scan scan = new Scan(); + Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, + CompareOp.NOT_EQUAL, filtered_val); + scan.setFilter(filter); + scan.setLoadColumnFamiliesOnDemand(true); + InternalScanner s = region.getScanner(scan); - assertTrue(s.next(results)); - assertEquals(results.size(), 3); - assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha)); - assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential)); - assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined)); - results.clear(); + List results = new ArrayList(); + assertTrue(s.next(results)); + assertEquals(results.size(), 1); + results.clear(); - assertFalse(s.next(results)); - assertEquals(results.size(), 0); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + assertTrue(s.next(results)); + assertEquals(results.size(), 3); + assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha)); + assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential)); + assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined)); + results.clear(); + + assertFalse(s.next(results)); + assertEquals(results.size(), 0); } /** @@ -3505,88 +3283,83 @@ public class TestHRegion { final byte[] cf_second = Bytes.toBytes("second"); this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second); - try { - final byte[] col_a = Bytes.toBytes("a"); - final byte[] col_b = Bytes.toBytes("b"); - - Put put; - - for (int i = 0; i < 10; i++) { - put = new Put(Bytes.toBytes("r" + Integer.toString(i))); - put.add(cf_first, col_a, Bytes.toBytes(i)); - if (i < 5) { - put.add(cf_first, col_b, Bytes.toBytes(i)); - put.add(cf_second, col_a, Bytes.toBytes(i)); - put.add(cf_second, col_b, Bytes.toBytes(i)); - } - region.put(put); + final byte[] col_a = Bytes.toBytes("a"); + final byte[] col_b = Bytes.toBytes("b"); + + Put put; + + for (int i = 0; i < 10; i++) { + put = new Put(Bytes.toBytes("r" + Integer.toString(i))); + put.add(cf_first, col_a, Bytes.toBytes(i)); + if (i < 5) { + put.add(cf_first, col_b, Bytes.toBytes(i)); + put.add(cf_second, col_a, Bytes.toBytes(i)); + put.add(cf_second, col_b, Bytes.toBytes(i)); } + region.put(put); + } - Scan scan = new Scan(); - scan.setLoadColumnFamiliesOnDemand(true); - Filter bogusFilter = new FilterBase() { - @Override - public ReturnCode filterKeyValue(Cell ignored) throws IOException { - return ReturnCode.INCLUDE; - } - @Override - public boolean isFamilyEssential(byte[] name) { - return Bytes.equals(name, cf_first); - } - }; - - scan.setFilter(bogusFilter); - InternalScanner s = region.getScanner(scan); - - // Our data looks like this: - // r0: first:a, first:b, second:a, second:b - // r1: first:a, first:b, second:a, second:b - // r2: first:a, first:b, second:a, second:b - // r3: first:a, first:b, second:a, second:b - // r4: first:a, first:b, second:a, second:b - // r5: first:a - // r6: first:a - // r7: first:a - // r8: first:a - // r9: first:a - - // But due to next's limit set to 3, we should get this: - // r0: first:a, first:b, second:a - // r0: second:b - // r1: first:a, first:b, second:a - // r1: second:b - // r2: first:a, first:b, second:a - // r2: second:b - // r3: first:a, first:b, second:a - // r3: second:b - // r4: first:a, first:b, second:a - // r4: second:b - // r5: first:a - // r6: first:a - // r7: first:a - // r8: first:a - // r9: first:a - - List results = new ArrayList(); - int index = 0; - ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build(); - while (true) { - boolean more = s.next(results, scannerContext); - if ((index >> 1) < 5) { - if (index % 2 == 0) - assertEquals(results.size(), 3); - else - assertEquals(results.size(), 1); - } else - assertEquals(results.size(), 1); - results.clear(); - index++; - if (!more) - break; + Scan scan = new Scan(); + scan.setLoadColumnFamiliesOnDemand(true); + Filter bogusFilter = new FilterBase() { + @Override + public ReturnCode filterKeyValue(Cell ignored) throws IOException { + return ReturnCode.INCLUDE; } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + @Override + public boolean isFamilyEssential(byte[] name) { + return Bytes.equals(name, cf_first); + } + }; + + scan.setFilter(bogusFilter); + InternalScanner s = region.getScanner(scan); + + // Our data looks like this: + // r0: first:a, first:b, second:a, second:b + // r1: first:a, first:b, second:a, second:b + // r2: first:a, first:b, second:a, second:b + // r3: first:a, first:b, second:a, second:b + // r4: first:a, first:b, second:a, second:b + // r5: first:a + // r6: first:a + // r7: first:a + // r8: first:a + // r9: first:a + + // But due to next's limit set to 3, we should get this: + // r0: first:a, first:b, second:a + // r0: second:b + // r1: first:a, first:b, second:a + // r1: second:b + // r2: first:a, first:b, second:a + // r2: second:b + // r3: first:a, first:b, second:a + // r3: second:b + // r4: first:a, first:b, second:a + // r4: second:b + // r5: first:a + // r6: first:a + // r7: first:a + // r8: first:a + // r9: first:a + + List results = new ArrayList(); + int index = 0; + ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build(); + while (true) { + boolean more = s.next(results, scannerContext); + if ((index >> 1) < 5) { + if (index % 2 == 0) + assertEquals(results.size(), 3); + else + assertEquals(results.size(), 1); + } else + assertEquals(results.size(), 1); + results.clear(); + index++; + if (!more) + break; } } @@ -3611,9 +3384,8 @@ public class TestHRegion { region.put(p); } region.flush(false); - HRegion.closeHRegion(this.region); - this.region = null; } + // //////////////////////////////////////////////////////////////////////////// // Split test // //////////////////////////////////////////////////////////////////////////// @@ -3631,81 +3403,76 @@ public class TestHRegion { String method = this.getName(); this.region = initHRegion(tableName, method, hc, families); + LOG.info("" + HBaseTestCase.addContent(region, fam3)); + region.flush(true); + region.compactStores(); + byte[] splitRow = region.checkSplit(); + assertNotNull(splitRow); + LOG.info("SplitRow: " + Bytes.toString(splitRow)); + HRegion[] regions = splitRegion(region, splitRow); try { - LOG.info("" + HBaseTestCase.addContent(region, fam3)); - region.flush(true); - region.compactStores(); - byte[] splitRow = region.checkSplit(); - assertNotNull(splitRow); - LOG.info("SplitRow: " + Bytes.toString(splitRow)); - HRegion[] regions = splitRegion(region, splitRow); - try { - // Need to open the regions. - // TODO: Add an 'open' to HRegion... don't do open by constructing - // instance. - for (int i = 0; i < regions.length; i++) { - regions[i] = HRegion.openHRegion(regions[i], null); - } - // Assert can get rows out of new regions. Should be able to get first - // row from first region and the midkey from second region. - assertGet(regions[0], fam3, Bytes.toBytes(START_KEY)); - assertGet(regions[1], fam3, splitRow); - // Test I can get scanner and that it starts at right place. - assertScan(regions[0], fam3, Bytes.toBytes(START_KEY)); - assertScan(regions[1], fam3, splitRow); - // Now prove can't split regions that have references. - for (int i = 0; i < regions.length; i++) { - // Add so much data to this region, we create a store file that is > - // than one of our unsplitable references. it will. - for (int j = 0; j < 2; j++) { - HBaseTestCase.addContent(regions[i], fam3); - } - HBaseTestCase.addContent(regions[i], fam2); - HBaseTestCase.addContent(regions[i], fam1); - regions[i].flush(true); + // Need to open the regions. + // TODO: Add an 'open' to HRegion... don't do open by constructing + // instance. + for (int i = 0; i < regions.length; i++) { + regions[i] = HRegion.openHRegion(regions[i], null); + } + // Assert can get rows out of new regions. Should be able to get first + // row from first region and the midkey from second region. + assertGet(regions[0], fam3, Bytes.toBytes(START_KEY)); + assertGet(regions[1], fam3, splitRow); + // Test I can get scanner and that it starts at right place. + assertScan(regions[0], fam3, Bytes.toBytes(START_KEY)); + assertScan(regions[1], fam3, splitRow); + // Now prove can't split regions that have references. + for (int i = 0; i < regions.length; i++) { + // Add so much data to this region, we create a store file that is > + // than one of our unsplitable references. it will. + for (int j = 0; j < 2; j++) { + HBaseTestCase.addContent(regions[i], fam3); } + HBaseTestCase.addContent(regions[i], fam2); + HBaseTestCase.addContent(regions[i], fam1); + regions[i].flush(true); + } - byte[][] midkeys = new byte[regions.length][]; - // To make regions splitable force compaction. - for (int i = 0; i < regions.length; i++) { - regions[i].compactStores(); - midkeys[i] = regions[i].checkSplit(); - } + byte[][] midkeys = new byte[regions.length][]; + // To make regions splitable force compaction. + for (int i = 0; i < regions.length; i++) { + regions[i].compactStores(); + midkeys[i] = regions[i].checkSplit(); + } - TreeMap sortedMap = new TreeMap(); - // Split these two daughter regions so then I'll have 4 regions. Will - // split because added data above. - for (int i = 0; i < regions.length; i++) { - HRegion[] rs = null; - if (midkeys[i] != null) { - rs = splitRegion(regions[i], midkeys[i]); - for (int j = 0; j < rs.length; j++) { - sortedMap.put(Bytes.toString(rs[j].getRegionInfo().getRegionName()), - HRegion.openHRegion(rs[j], null)); - } - } - } - LOG.info("Made 4 regions"); - // The splits should have been even. Test I can get some arbitrary row - // out of each. - int interval = (LAST_CHAR - FIRST_CHAR) / 3; - byte[] b = Bytes.toBytes(START_KEY); - for (HRegion r : sortedMap.values()) { - assertGet(r, fam3, b); - b[0] += interval; - } - } finally { - for (int i = 0; i < regions.length; i++) { - try { - regions[i].close(); - } catch (IOException e) { - // Ignore. + TreeMap sortedMap = new TreeMap(); + // Split these two daughter regions so then I'll have 4 regions. Will + // split because added data above. + for (int i = 0; i < regions.length; i++) { + HRegion[] rs = null; + if (midkeys[i] != null) { + rs = splitRegion(regions[i], midkeys[i]); + for (int j = 0; j < rs.length; j++) { + sortedMap.put(Bytes.toString(rs[j].getRegionInfo().getRegionName()), + HRegion.openHRegion(rs[j], null)); } } } + LOG.info("Made 4 regions"); + // The splits should have been even. Test I can get some arbitrary row + // out of each. + int interval = (LAST_CHAR - FIRST_CHAR) / 3; + byte[] b = Bytes.toBytes(START_KEY); + for (HRegion r : sortedMap.values()) { + assertGet(r, fam3, b); + b[0] += interval; + } } finally { - HRegion.closeHRegion(this.region); - this.region = null; + for (int i = 0; i < regions.length; i++) { + try { + regions[i].close(); + } catch (IOException e) { + // Ignore. + } + } } } @@ -3728,24 +3495,18 @@ public class TestHRegion { region.flush(true); HRegion[] regions = null; - try { - regions = splitRegion(region, Bytes.toBytes("" + splitRow)); - // Opening the regions returned. - for (int i = 0; i < regions.length; i++) { - regions[i] = HRegion.openHRegion(regions[i], null); - } - // Verifying that the region has been split - assertEquals(2, regions.length); - - // Verifying that all data is still there and that data is in the right - // place - verifyData(regions[0], startRow, numRows, qualifier, families); - verifyData(regions[1], splitRow, numRows, qualifier, families); - - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + regions = splitRegion(region, Bytes.toBytes("" + splitRow)); + // Opening the regions returned. + for (int i = 0; i < regions.length; i++) { + regions[i] = HRegion.openHRegion(regions[i], null); } + // Verifying that the region has been split + assertEquals(2, regions.length); + + // Verifying that all data is still there and that data is in the right + // place + verifyData(regions[0], startRow, numRows, qualifier, families); + verifyData(regions[1], splitRow, numRows, qualifier, families); } @Test @@ -3768,52 +3529,46 @@ public class TestHRegion { region.flush(true); HRegion[] regions = null; - try { - // Set force split - region.forceSplit(splitRowBytes); - assertTrue(region.shouldForceSplit()); - // Split point should be the force split row - assertTrue(Bytes.equals(splitRowBytes, region.checkSplit())); - - // Add a store that has references. - HStore storeMock = Mockito.mock(HStore.class); - when(storeMock.hasReferences()).thenReturn(true); - when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf")); - when(storeMock.close()).thenReturn(ImmutableList.of()); - when(storeMock.getColumnFamilyName()).thenReturn("cf"); - region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock); - assertTrue(region.hasReferences()); - - // Will not split since the store has references. - regions = splitRegion(region, splitRowBytes); - assertNull(regions); - - // Region force split should be cleared after the split try. - assertFalse(region.shouldForceSplit()); - - // Remove the store that has references. - region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName())); - assertFalse(region.hasReferences()); - - // Now we can split. - regions = splitRegion(region, splitRowBytes); - - // Opening the regions returned. - for (int i = 0; i < regions.length; i++) { - regions[i] = HRegion.openHRegion(regions[i], null); - } - // Verifying that the region has been split - assertEquals(2, regions.length); + // Set force split + region.forceSplit(splitRowBytes); + assertTrue(region.shouldForceSplit()); + // Split point should be the force split row + assertTrue(Bytes.equals(splitRowBytes, region.checkSplit())); - // Verifying that all data is still there and that data is in the right - // place - verifyData(regions[0], startRow, numRows, qualifier, families); - verifyData(regions[1], splitRow, numRows, qualifier, families); + // Add a store that has references. + HStore storeMock = Mockito.mock(HStore.class); + when(storeMock.hasReferences()).thenReturn(true); + when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf")); + when(storeMock.close()).thenReturn(ImmutableList.of()); + when(storeMock.getColumnFamilyName()).thenReturn("cf"); + region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock); + assertTrue(region.hasReferences()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // Will not split since the store has references. + regions = splitRegion(region, splitRowBytes); + assertNull(regions); + + // Region force split should be cleared after the split try. + assertFalse(region.shouldForceSplit()); + + // Remove the store that has references. + region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName())); + assertFalse(region.hasReferences()); + + // Now we can split. + regions = splitRegion(region, splitRowBytes); + + // Opening the regions returned. + for (int i = 0; i < regions.length; i++) { + regions[i] = HRegion.openHRegion(regions[i], null); } + // Verifying that the region has been split + assertEquals(2, regions.length); + + // Verifying that all data is still there and that data is in the right + // place + verifyData(regions[0], startRow, numRows, qualifier, families); + verifyData(regions[1], splitRow, numRows, qualifier, families); } /** @@ -4243,22 +3998,17 @@ public class TestHRegion { byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family); + byte[] rowNotServed = Bytes.toBytes("a"); + Get g = new Get(rowNotServed); try { - byte[] rowNotServed = Bytes.toBytes("a"); - Get g = new Get(rowNotServed); - try { - region.get(g); - fail(); - } catch (WrongRegionException x) { - // OK - } - byte[] row = Bytes.toBytes("y"); - g = new Get(row); region.get(g); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + fail(); + } catch (WrongRegionException x) { + // OK } + byte[] row = Bytes.toBytes("y"); + g = new Get(row); + region.get(g); } @Test @@ -4268,36 +4018,32 @@ public class TestHRegion { // Setting up region String method = "testIndexesScanWithOneDeletedRow"; this.region = initHRegion(tableName, method, CONF, family); - try { - Put put = new Put(Bytes.toBytes(1L)); - put.add(family, qual1, 1L, Bytes.toBytes(1L)); - region.put(put); + Put put = new Put(Bytes.toBytes(1L)); + put.add(family, qual1, 1L, Bytes.toBytes(1L)); + region.put(put); - region.flush(true); + region.flush(true); - Delete delete = new Delete(Bytes.toBytes(1L), 1L); - region.delete(delete); + Delete delete = new Delete(Bytes.toBytes(1L), 1L); + region.delete(delete); - put = new Put(Bytes.toBytes(2L)); - put.add(family, qual1, 2L, Bytes.toBytes(2L)); - region.put(put); + put = new Put(Bytes.toBytes(2L)); + put.add(family, qual1, 2L, Bytes.toBytes(2L)); + region.put(put); - Scan idxScan = new Scan(); - idxScan.addFamily(family); - idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays. asList( - new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1, - CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L)))))); - InternalScanner scanner = region.getScanner(idxScan); - List res = new ArrayList(); + Scan idxScan = new Scan(); + idxScan.addFamily(family); + idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays. asList( + new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1, + CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L)))))); + InternalScanner scanner = region.getScanner(idxScan); + List res = new ArrayList<>(); - while (scanner.next(res)) - ; - assertEquals(1L, res.size()); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + while (scanner.next(res)) { + // Ignore res value. } + assertEquals(1L, res.size()); } // //////////////////////////////////////////////////////////////////////////// @@ -4316,48 +4062,43 @@ public class TestHRegion { htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); this.region = TEST_UTIL.createLocalHRegion(info, htd); - try { - int num_unique_rows = 10; - int duplicate_multiplier = 2; - int num_storefiles = 4; - - int version = 0; - for (int f = 0; f < num_storefiles; f++) { - for (int i = 0; i < duplicate_multiplier; i++) { - for (int j = 0; j < num_unique_rows; j++) { - Put put = new Put(Bytes.toBytes("row" + j)); - put.setDurability(Durability.SKIP_WAL); - put.add(fam1, qf1, version++, val1); - region.put(put); - } + int num_unique_rows = 10; + int duplicate_multiplier = 2; + int num_storefiles = 4; + + int version = 0; + for (int f = 0; f < num_storefiles; f++) { + for (int i = 0; i < duplicate_multiplier; i++) { + for (int j = 0; j < num_unique_rows; j++) { + Put put = new Put(Bytes.toBytes("row" + j)); + put.setDurability(Durability.SKIP_WAL); + put.add(fam1, qf1, version++, val1); + region.put(put); } - region.flush(true); - } - // before compaction - HStore store = (HStore) region.getStore(fam1); - Collection storeFiles = store.getStorefiles(); - for (StoreFile storefile : storeFiles) { - StoreFile.Reader reader = storefile.getReader(); - reader.loadFileInfo(); - reader.loadBloomfilter(); - assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries()); - assertEquals(num_unique_rows, reader.getFilterEntries()); } + region.flush(true); + } + // before compaction + HStore store = (HStore) region.getStore(fam1); + Collection storeFiles = store.getStorefiles(); + for (StoreFile storefile : storeFiles) { + StoreFile.Reader reader = storefile.getReader(); + reader.loadFileInfo(); + reader.loadBloomfilter(); + assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries()); + assertEquals(num_unique_rows, reader.getFilterEntries()); + } - region.compact(true); + region.compact(true); - // after compaction - storeFiles = store.getStorefiles(); - for (StoreFile storefile : storeFiles) { - StoreFile.Reader reader = storefile.getReader(); - reader.loadFileInfo(); - reader.loadBloomfilter(); - assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries()); - assertEquals(num_unique_rows, reader.getFilterEntries()); - } - } finally { - HRegion.closeHRegion(this.region); - this.region = null; + // after compaction + storeFiles = store.getStorefiles(); + for (StoreFile storefile : storeFiles) { + StoreFile.Reader reader = storefile.getReader(); + reader.loadFileInfo(); + reader.loadBloomfilter(); + assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries()); + assertEquals(num_unique_rows, reader.getFilterEntries()); } } @@ -4373,35 +4114,30 @@ public class TestHRegion { htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); this.region = TEST_UTIL.createLocalHRegion(info, htd); - try { - // For row:0, col:0: insert versions 1 through 5. - byte row[] = Bytes.toBytes("row:" + 0); - byte column[] = Bytes.toBytes("column:" + 0); - Put put = new Put(row); - put.setDurability(Durability.SKIP_WAL); - for (long idx = 1; idx <= 4; idx++) { - put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); - } - region.put(put); + // For row:0, col:0: insert versions 1 through 5. + byte row[] = Bytes.toBytes("row:" + 0); + byte column[] = Bytes.toBytes("column:" + 0); + Put put = new Put(row); + put.setDurability(Durability.SKIP_WAL); + for (long idx = 1; idx <= 4; idx++) { + put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); + } + region.put(put); - // Flush - region.flush(true); + // Flush + region.flush(true); - // Get rows - Get get = new Get(row); - get.setMaxVersions(); - Cell[] kvs = region.get(get).rawCells(); - - // Check if rows are correct - assertEquals(4, kvs.length); - checkOneCell(kvs[0], FAMILY, 0, 0, 4); - checkOneCell(kvs[1], FAMILY, 0, 0, 3); - checkOneCell(kvs[2], FAMILY, 0, 0, 2); - checkOneCell(kvs[3], FAMILY, 0, 0, 1); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // Get rows + Get get = new Get(row); + get.setMaxVersions(); + Cell[] kvs = region.get(get).rawCells(); + + // Check if rows are correct + assertEquals(4, kvs.length); + checkOneCell(kvs[0], FAMILY, 0, 0, 4); + checkOneCell(kvs[1], FAMILY, 0, 0, 3); + checkOneCell(kvs[2], FAMILY, 0, 0, 2); + checkOneCell(kvs[3], FAMILY, 0, 0, 1); } /** @@ -4421,30 +4157,25 @@ public class TestHRegion { htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); this.region = TEST_UTIL.createLocalHRegion(info, htd); - try { - // Insert some data - byte row[] = Bytes.toBytes("row1"); - byte col[] = Bytes.toBytes("col1"); + // Insert some data + byte row[] = Bytes.toBytes("row1"); + byte col[] = Bytes.toBytes("col1"); - Put put = new Put(row); - put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); - region.put(put); - region.flush(true); + Put put = new Put(row); + put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); + region.put(put); + region.flush(true); - Delete del = new Delete(row); - region.delete(del); - region.flush(true); + Delete del = new Delete(row); + region.delete(del); + region.flush(true); - // Get remaining rows (should have none) - Get get = new Get(row); - get.addColumn(familyName, col); + // Get remaining rows (should have none) + Get get = new Get(row); + get.addColumn(familyName, col); - Cell[] keyValues = region.get(get).rawCells(); - assertTrue(keyValues.length == 0); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + Cell[] keyValues = region.get(get).rawCells(); + assertEquals(0,keyValues.length); } @Test @@ -4551,8 +4282,6 @@ public class TestHRegion { break; } } - } finally { - HRegion.closeHRegion(region); } } @@ -4570,7 +4299,7 @@ public class TestHRegion { HRegionInfo hri = new HRegionInfo(htd.getTableName()); // Create a region and skip the initialization (like CreateTableHandler) - HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true); + region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true); // HRegion region = TEST_UTIL.createLocalHRegion(hri, htd); Path regionDir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); @@ -4690,7 +4419,6 @@ public class TestHRegion { assertEquals(kvs.size(), 1); Cell kv = kvs.get(0); assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset())); - this.region = null; } /** @@ -4783,7 +4511,6 @@ public class TestHRegion { byte[] appendResult = new byte[kv.getValueLength()]; System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength()); assertArrayEquals(expected, appendResult); - this.region = null; } /** @@ -5357,48 +5084,43 @@ public class TestHRegion { long ts = 1; String method = this.getName(); this.region = initHRegion(tableName, method, families); - try { - KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, - null); - KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); - KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); - Put put = null; - put = new Put(rowC); - put.add(kv1); - put.add(kv11); - region.put(put); - put = new Put(rowA); - put.add(kv2); - region.put(put); - put = new Put(rowB); - put.add(kv3); - region.put(put); + KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, + null); + KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); + KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); + Put put = null; + put = new Put(rowC); + put.add(kv1); + put.add(kv11); + region.put(put); + put = new Put(rowA); + put.add(kv2); + region.put(put); + put = new Put(rowB); + put.add(kv3); + region.put(put); - Scan scan = new Scan(rowC); - scan.setMaxVersions(5); - scan.setReversed(true); - InternalScanner scanner = region.getScanner(scan); - List currRow = new ArrayList(); - boolean hasNext = scanner.next(currRow); - assertEquals(2, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA)); - assertFalse(hasNext); - scanner.close(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + Scan scan = new Scan(rowC); + scan.setMaxVersions(5); + scan.setReversed(true); + InternalScanner scanner = region.getScanner(scan); + List currRow = new ArrayList(); + boolean hasNext = scanner.next(currRow); + assertEquals(2, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA)); + assertFalse(hasNext); + scanner.close(); } @Test (timeout=60000) @@ -5414,48 +5136,43 @@ public class TestHRegion { long ts = 1; String method = this.getName(); this.region = initHRegion(tableName, method, families); - try { - KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, - null); - KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); - KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); - Put put = null; - put = new Put(rowC); - put.add(kv1); - put.add(kv11); - region.put(put); - put = new Put(rowA); - put.add(kv2); - region.put(put); - put = new Put(rowB); - put.add(kv3); - region.put(put); + KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, + null); + KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); + KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); + Put put = null; + put = new Put(rowC); + put.add(kv1); + put.add(kv11); + region.put(put); + put = new Put(rowA); + put.add(kv2); + region.put(put); + put = new Put(rowB); + put.add(kv3); + region.put(put); - Scan scan = new Scan(rowD); - List currRow = new ArrayList(); - scan.setReversed(true); - scan.setMaxVersions(5); - InternalScanner scanner = region.getScanner(scan); - boolean hasNext = scanner.next(currRow); - assertEquals(2, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA)); - assertFalse(hasNext); - scanner.close(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + Scan scan = new Scan(rowD); + List currRow = new ArrayList(); + scan.setReversed(true); + scan.setMaxVersions(5); + InternalScanner scanner = region.getScanner(scan); + boolean hasNext = scanner.next(currRow); + assertEquals(2, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA)); + assertFalse(hasNext); + scanner.close(); } @Test (timeout=60000) @@ -5466,50 +5183,45 @@ public class TestHRegion { byte[] rowB = Bytes.toBytes("rowB"); byte[] cf = Bytes.toBytes("CF"); byte[][] families = { cf }; - byte[] col = Bytes.toBytes("C"); - long ts = 1; - String method = this.getName(); - this.region = initHRegion(tableName, method, families); - try { - KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); - KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, - null); - KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); - KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); - Put put = null; - put = new Put(rowC); - put.add(kv1); - put.add(kv11); - region.put(put); - put = new Put(rowA); - put.add(kv2); - region.put(put); - put = new Put(rowB); - put.add(kv3); - region.put(put); - Scan scan = new Scan(); - List currRow = new ArrayList(); - scan.setReversed(true); - InternalScanner scanner = region.getScanner(scan); - boolean hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA)); - assertFalse(hasNext); - scanner.close(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + byte[] col = Bytes.toBytes("C"); + long ts = 1; + String method = this.getName(); + this.region = initHRegion(tableName, method, families); + KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, + null); + KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); + KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); + Put put = null; + put = new Put(rowC); + put.add(kv1); + put.add(kv11); + region.put(put); + put = new Put(rowA); + put.add(kv2); + region.put(put); + put = new Put(rowB); + put.add(kv3); + region.put(put); + Scan scan = new Scan(); + List currRow = new ArrayList(); + scan.setReversed(true); + InternalScanner scanner = region.getScanner(scan); + boolean hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA)); + assertFalse(hasNext); + scanner.close(); } @Test (timeout=60000) @@ -5527,67 +5239,62 @@ public class TestHRegion { long ts = 1; String method = this.getName(); this.region = initHRegion(tableName, method, families); - try { - KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); - KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); - Put put = null; - put = new Put(rowA); - put.add(kv1); - region.put(put); - put = new Put(rowB); - put.add(kv2); - region.put(put); - put = new Put(rowC); - put.add(kv3); - region.put(put); - put = new Put(rowD); - put.add(kv4_1); - region.put(put); - put = new Put(rowD); - put.add(kv4_2); - region.put(put); - put = new Put(rowE); - put.add(kv5); - region.put(put); - region.flush(true); - Scan scan = new Scan(rowD, rowA); - scan.addColumn(families[0], col1); - scan.setReversed(true); - List currRow = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); - boolean hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); - assertFalse(hasNext); - scanner.close(); - - scan = new Scan(rowD, rowA); - scan.addColumn(families[0], col2); - scan.setReversed(true); - currRow.clear(); - scanner = region.getScanner(scan); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); - scanner.close(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); + KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); + Put put = null; + put = new Put(rowA); + put.add(kv1); + region.put(put); + put = new Put(rowB); + put.add(kv2); + region.put(put); + put = new Put(rowC); + put.add(kv3); + region.put(put); + put = new Put(rowD); + put.add(kv4_1); + region.put(put); + put = new Put(rowD); + put.add(kv4_2); + region.put(put); + put = new Put(rowE); + put.add(kv5); + region.put(put); + region.flush(true); + Scan scan = new Scan(rowD, rowA); + scan.addColumn(families[0], col1); + scan.setReversed(true); + List currRow = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + boolean hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); + assertFalse(hasNext); + scanner.close(); + + scan = new Scan(rowD, rowA); + scan.addColumn(families[0], col2); + scan.setReversed(true); + currRow.clear(); + scanner = region.getScanner(scan); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); + scanner.close(); } @Test (timeout=60000) @@ -5607,67 +5314,62 @@ public class TestHRegion { HBaseConfiguration config = new HBaseConfiguration(); config.setInt("test.block.size", 1); this.region = initHRegion(tableName, method, config, families); - try { - KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); - KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); - KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); - Put put = null; - put = new Put(rowA); - put.add(kv1); - region.put(put); - put = new Put(rowB); - put.add(kv2); - region.put(put); - put = new Put(rowC); - put.add(kv3); - region.put(put); - put = new Put(rowD); - put.add(kv4_1); - region.put(put); - put = new Put(rowD); - put.add(kv4_2); - region.put(put); - put = new Put(rowE); - put.add(kv5); - region.put(put); - region.flush(true); - Scan scan = new Scan(rowD, rowA); - scan.addColumn(families[0], col1); - scan.setReversed(true); - List currRow = new ArrayList(); - InternalScanner scanner = region.getScanner(scan); - boolean hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); - assertFalse(hasNext); - scanner.close(); - - scan = new Scan(rowD, rowA); - scan.addColumn(families[0], col2); - scan.setReversed(true); - currRow.clear(); - scanner = region.getScanner(scan); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); - scanner.close(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); + KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); + KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); + Put put = null; + put = new Put(rowA); + put.add(kv1); + region.put(put); + put = new Put(rowB); + put.add(kv2); + region.put(put); + put = new Put(rowC); + put.add(kv3); + region.put(put); + put = new Put(rowD); + put.add(kv4_1); + region.put(put); + put = new Put(rowD); + put.add(kv4_2); + region.put(put); + put = new Put(rowE); + put.add(kv5); + region.put(put); + region.flush(true); + Scan scan = new Scan(rowD, rowA); + scan.addColumn(families[0], col1); + scan.setReversed(true); + List currRow = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + boolean hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB)); + assertFalse(hasNext); + scanner.close(); + + scan = new Scan(rowD, rowA); + scan.addColumn(families[0], col2); + scan.setReversed(true); + currRow.clear(); + scanner = region.getScanner(scan); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD)); + scanner.close(); } @Test (timeout=60000) @@ -5690,145 +5392,140 @@ public class TestHRegion { // disable compactions in this test. conf.setInt("hbase.hstore.compactionThreshold", 10000); this.region = initHRegion(tableName, method, conf, families); - try { - // kv naming style: kv(row number) totalKvCountInThisRow seq no - KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, - null); - KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, - null); - KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, - KeyValue.Type.Put, null); - KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, - null); - KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, - null); - KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, - null); - KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, - KeyValue.Type.Put, null); - KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, - null); - KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, - KeyValue.Type.Put, null); - KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, - null); - KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, - null); - KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, - KeyValue.Type.Put, null); - KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, - null); - KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, - KeyValue.Type.Put, null); - KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, - null); - KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, - null); - // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv) - Put put = null; - put = new Put(row1); - put.add(kv1_2_1); - region.put(put); - put = new Put(row2); - put.add(kv2_4_1); - region.put(put); - put = new Put(row4); - put.add(kv4_5_4); - put.add(kv4_5_5); - region.put(put); - region.flush(true); - // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv) - put = new Put(row4); - put.add(kv4_5_1); - put.add(kv4_5_3); - region.put(put); - put = new Put(row1); - put.add(kv1_2_2); - region.put(put); - put = new Put(row2); - put.add(kv2_4_4); - region.put(put); - region.flush(true); - // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv) - put = new Put(row4); - put.add(kv4_5_2); - region.put(put); - put = new Put(row2); - put.add(kv2_4_2); - put.add(kv2_4_3); - region.put(put); - put = new Put(row3); - put.add(kv3_2_2); - region.put(put); - region.flush(true); - // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max) - // ( 2 kv) - put = new Put(row0); - put.add(kv0_1_1); - region.put(put); - put = new Put(row3); - put.add(kv3_2_1); - region.put(put); - put = new Put(row5); - put.add(kv5_2_1); - put.add(kv5_2_2); - region.put(put); - // scan range = ["row4", min), skip the max "row5" - Scan scan = new Scan(row4); - scan.setMaxVersions(5); - scan.setBatch(3); - scan.setReversed(true); - InternalScanner scanner = region.getScanner(scan); - List currRow = new ArrayList(); - boolean hasNext = false; - // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not - // included in scan range - // "row4" takes 2 next() calls since batch=3 - hasNext = scanner.next(currRow); - assertEquals(3, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row4)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(2, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row4)); - assertTrue(hasNext); - // 2. scan out "row3" (2 kv) - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(2, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row3)); - assertTrue(hasNext); - // 3. scan out "row2" (4 kvs) - // "row2" takes 2 next() calls since batch=3 - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(3, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row2)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row2)); - assertTrue(hasNext); - // 4. scan out "row1" (2 kv) - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(2, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row1)); - assertTrue(hasNext); - // 5. scan out "row0" (1 kv) - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row0)); - assertFalse(hasNext); - - scanner.close(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + // kv naming style: kv(row number) totalKvCountInThisRow seq no + KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, + null); + KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, + null); + KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, + KeyValue.Type.Put, null); + KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, + null); + KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, + null); + KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, + null); + KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, + KeyValue.Type.Put, null); + KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, + null); + KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, + KeyValue.Type.Put, null); + KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, + null); + KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, + null); + KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, + KeyValue.Type.Put, null); + KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, + null); + KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, + KeyValue.Type.Put, null); + KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, + null); + KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, + null); + // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv) + Put put = null; + put = new Put(row1); + put.add(kv1_2_1); + region.put(put); + put = new Put(row2); + put.add(kv2_4_1); + region.put(put); + put = new Put(row4); + put.add(kv4_5_4); + put.add(kv4_5_5); + region.put(put); + region.flush(true); + // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv) + put = new Put(row4); + put.add(kv4_5_1); + put.add(kv4_5_3); + region.put(put); + put = new Put(row1); + put.add(kv1_2_2); + region.put(put); + put = new Put(row2); + put.add(kv2_4_4); + region.put(put); + region.flush(true); + // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv) + put = new Put(row4); + put.add(kv4_5_2); + region.put(put); + put = new Put(row2); + put.add(kv2_4_2); + put.add(kv2_4_3); + region.put(put); + put = new Put(row3); + put.add(kv3_2_2); + region.put(put); + region.flush(true); + // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max) + // ( 2 kv) + put = new Put(row0); + put.add(kv0_1_1); + region.put(put); + put = new Put(row3); + put.add(kv3_2_1); + region.put(put); + put = new Put(row5); + put.add(kv5_2_1); + put.add(kv5_2_2); + region.put(put); + // scan range = ["row4", min), skip the max "row5" + Scan scan = new Scan(row4); + scan.setMaxVersions(5); + scan.setBatch(3); + scan.setReversed(true); + InternalScanner scanner = region.getScanner(scan); + List currRow = new ArrayList(); + boolean hasNext = false; + // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not + // included in scan range + // "row4" takes 2 next() calls since batch=3 + hasNext = scanner.next(currRow); + assertEquals(3, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row4)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(2, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row4)); + assertTrue(hasNext); + // 2. scan out "row3" (2 kv) + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(2, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row3)); + assertTrue(hasNext); + // 3. scan out "row2" (4 kvs) + // "row2" takes 2 next() calls since batch=3 + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(3, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row2)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row2)); + assertTrue(hasNext); + // 4. scan out "row1" (2 kv) + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(2, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row1)); + assertTrue(hasNext); + // 5. scan out "row0" (1 kv) + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row0)); + assertFalse(hasNext); + + scanner.close(); } @Test (timeout=60000) @@ -5850,59 +5547,54 @@ public class TestHRegion { // disable compactions in this test. conf.setInt("hbase.hstore.compactionThreshold", 10000); this.region = initHRegion(tableName, method, conf, families); - try { - KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null); - KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); - KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null); - KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null); - // storefile1 - Put put = new Put(row1); - put.add(kv1); - region.put(put); - region.flush(true); - // storefile2 - put = new Put(row2); - put.add(kv2); - region.put(put); - region.flush(true); - // storefile3 - put = new Put(row3); - put.add(kv3); - region.put(put); - region.flush(true); - // memstore - put = new Put(row4); - put.add(kv4); - region.put(put); - // scan range = ["row4", min) - Scan scan = new Scan(row4); - scan.setReversed(true); - scan.setBatch(10); - InternalScanner scanner = region.getScanner(scan); - List currRow = new ArrayList(); - boolean hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row4)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row3)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row2)); - assertTrue(hasNext); - currRow.clear(); - hasNext = scanner.next(currRow); - assertEquals(1, currRow.size()); - assertTrue(Bytes.equals(currRow.get(0).getRow(), row1)); - assertFalse(hasNext); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null); + KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); + KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null); + KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null); + // storefile1 + Put put = new Put(row1); + put.add(kv1); + region.put(put); + region.flush(true); + // storefile2 + put = new Put(row2); + put.add(kv2); + region.put(put); + region.flush(true); + // storefile3 + put = new Put(row3); + put.add(kv3); + region.put(put); + region.flush(true); + // memstore + put = new Put(row4); + put.add(kv4); + region.put(put); + // scan range = ["row4", min) + Scan scan = new Scan(row4); + scan.setReversed(true); + scan.setBatch(10); + InternalScanner scanner = region.getScanner(scan); + List currRow = new ArrayList(); + boolean hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row4)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row3)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row2)); + assertTrue(hasNext); + currRow.clear(); + hasNext = scanner.next(currRow); + assertEquals(1, currRow.size()); + assertTrue(Bytes.equals(currRow.get(0).getRow(), row1)); + assertFalse(hasNext); } /** @@ -5916,49 +5608,44 @@ public class TestHRegion { String method = this.getName(); HBaseConfiguration conf = new HBaseConfiguration(); this.region = initHRegion(tableName, method, conf, families); - try { - // setup with one storefile and one memstore, to create scanner and get an earlier readPt - Put put = new Put(Bytes.toBytes("19998")); - put.add(cf1, col, Bytes.toBytes("val")); - region.put(put); - region.flushcache(true, true); - Put put2 = new Put(Bytes.toBytes("19997")); - put2.add(cf1, col, Bytes.toBytes("val")); - region.put(put2); - - Scan scan = new Scan(Bytes.toBytes("19998")); - scan.setReversed(true); - InternalScanner scanner = region.getScanner(scan); - - // create one storefile contains many rows will be skipped - // to check StoreFileScanner.seekToPreviousRow - for (int i = 10000; i < 20000; i++) { - Put p = new Put(Bytes.toBytes("" + i)); - p.add(cf1, col, Bytes.toBytes("" + i)); - region.put(p); - } - region.flushcache(true, true); - - // create one memstore contains many rows will be skipped - // to check MemStoreScanner.seekToPreviousRow - for (int i = 10000; i < 20000; i++) { - Put p = new Put(Bytes.toBytes("" + i)); - p.add(cf1, col, Bytes.toBytes("" + i)); - region.put(p); - } + // setup with one storefile and one memstore, to create scanner and get an earlier readPt + Put put = new Put(Bytes.toBytes("19998")); + put.add(cf1, col, Bytes.toBytes("val")); + region.put(put); + region.flushcache(true, true); + Put put2 = new Put(Bytes.toBytes("19997")); + put2.add(cf1, col, Bytes.toBytes("val")); + region.put(put2); + + Scan scan = new Scan(Bytes.toBytes("19998")); + scan.setReversed(true); + InternalScanner scanner = region.getScanner(scan); + + // create one storefile contains many rows will be skipped + // to check StoreFileScanner.seekToPreviousRow + for (int i = 10000; i < 20000; i++) { + Put p = new Put(Bytes.toBytes("" + i)); + p.add(cf1, col, Bytes.toBytes("" + i)); + region.put(p); + } + region.flushcache(true, true); - List currRow = new ArrayList<>(); - boolean hasNext; - do { - hasNext = scanner.next(currRow); - } while (hasNext); - assertEquals(2, currRow.size()); - assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow()); - assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow()); - } finally { - HBaseTestingUtility.closeRegionAndWAL(this.region); - this.region = null; + // create one memstore contains many rows will be skipped + // to check MemStoreScanner.seekToPreviousRow + for (int i = 10000; i < 20000; i++) { + Put p = new Put(Bytes.toBytes("" + i)); + p.add(cf1, col, Bytes.toBytes("" + i)); + region.put(p); } + + List currRow = new ArrayList<>(); + boolean hasNext; + do { + hasNext = scanner.next(currRow); + } while (hasNext); + assertEquals(2, currRow.size()); + assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow()); + assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow()); } @Test (timeout=60000) @@ -5981,78 +5668,73 @@ public class TestHRegion { region.flush(true); HRegion [] regions = null; - try { - regions = splitRegion(region, Bytes.toBytes("" + splitRow)); - //Opening the regions returned. - for (int i = 0; i < regions.length; i++) { - regions[i] = HRegion.openHRegion(regions[i], null); - } - //Verifying that the region has been split - assertEquals(2, regions.length); - - //Verifying that all data is still there and that data is in the right - //place - verifyData(regions[0], startRow, numRows, qualifier, families); - verifyData(regions[1], splitRow, numRows, qualifier, families); - - //fire the reverse scan1: top range, and larger than the last row - Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows))); - scan.setReversed(true); - InternalScanner scanner = regions[1].getScanner(scan); - List currRow = new ArrayList(); - boolean more = false; - int verify = startRow + 2 * numRows - 1; - do { - more = scanner.next(currRow); - assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); - verify--; - currRow.clear(); - } while(more); - assertEquals(verify, startRow + numRows - 1); - scanner.close(); - //fire the reverse scan2: top range, and equals to the last row - scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1))); - scan.setReversed(true); - scanner = regions[1].getScanner(scan); - verify = startRow + 2 * numRows - 1; - do { - more = scanner.next(currRow); - assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); - verify--; - currRow.clear(); - } while(more); - assertEquals(verify, startRow + numRows - 1); - scanner.close(); - //fire the reverse scan3: bottom range, and larger than the last row - scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows))); - scan.setReversed(true); - scanner = regions[0].getScanner(scan); - verify = startRow + numRows - 1; - do { - more = scanner.next(currRow); - assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); - verify--; - currRow.clear(); - } while(more); - assertEquals(verify, 99); - scanner.close(); - //fire the reverse scan4: bottom range, and equals to the last row - scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1))); - scan.setReversed(true); - scanner = regions[0].getScanner(scan); - verify = startRow + numRows - 1; - do { - more = scanner.next(currRow); - assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); - verify--; - currRow.clear(); - } while(more); - assertEquals(verify, startRow - 1); - scanner.close(); - } finally { - HRegion.closeHRegion(this.region); - this.region = null; - } + regions = splitRegion(region, Bytes.toBytes("" + splitRow)); + //Opening the regions returned. + for (int i = 0; i < regions.length; i++) { + regions[i] = HRegion.openHRegion(regions[i], null); + } + //Verifying that the region has been split + assertEquals(2, regions.length); + + //Verifying that all data is still there and that data is in the right + //place + verifyData(regions[0], startRow, numRows, qualifier, families); + verifyData(regions[1], splitRow, numRows, qualifier, families); + + //fire the reverse scan1: top range, and larger than the last row + Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows))); + scan.setReversed(true); + InternalScanner scanner = regions[1].getScanner(scan); + List currRow = new ArrayList(); + boolean more = false; + int verify = startRow + 2 * numRows - 1; + do { + more = scanner.next(currRow); + assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); + verify--; + currRow.clear(); + } while(more); + assertEquals(verify, startRow + numRows - 1); + scanner.close(); + //fire the reverse scan2: top range, and equals to the last row + scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1))); + scan.setReversed(true); + scanner = regions[1].getScanner(scan); + verify = startRow + 2 * numRows - 1; + do { + more = scanner.next(currRow); + assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); + verify--; + currRow.clear(); + } while(more); + assertEquals(verify, startRow + numRows - 1); + scanner.close(); + //fire the reverse scan3: bottom range, and larger than the last row + scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows))); + scan.setReversed(true); + scanner = regions[0].getScanner(scan); + verify = startRow + numRows - 1; + do { + more = scanner.next(currRow); + assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); + verify--; + currRow.clear(); + } while(more); + assertEquals(verify, 99); + scanner.close(); + //fire the reverse scan4: bottom range, and equals to the last row + scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1))); + scan.setReversed(true); + scanner = regions[0].getScanner(scan); + verify = startRow + numRows - 1; + do { + more = scanner.next(currRow); + assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + ""); + verify--; + currRow.clear(); + } while(more); + assertEquals(verify, startRow - 1); + scanner.close(); } @Test @@ -6076,9 +5758,6 @@ public class TestHRegion { region.delete(new Delete(row)); Assert.assertEquals(4L, region.getWriteRequestsCount()); - - HRegion.closeHRegion(this.region); - this.region = null; } @Test @@ -6095,9 +5774,8 @@ public class TestHRegion { HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY); // open the region w/o rss and wal and flush some files - HRegion region = - HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL - .getConfiguration(), htd); + region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), + htd); assertNotNull(region); // create a file in fam1 for the region before opening in OpenRegionHandler @@ -6111,44 +5789,39 @@ public class TestHRegion { WAL wal = mockWAL(); when(rss.getWAL((HRegionInfo) any())).thenReturn(wal); - try { - region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), - TEST_UTIL.getConfiguration(), rss, null); - - verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any() - , editCaptor.capture(), anyBoolean()); - - WALEdit edit = editCaptor.getValue(); - assertNotNull(edit); - assertNotNull(edit.getCells()); - assertEquals(1, edit.getCells().size()); - RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); - assertNotNull(desc); - - LOG.info("RegionEventDescriptor from WAL: " + desc); - - assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); - assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName())); - assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(), - hri.getEncodedNameAsBytes())); - assertTrue(desc.getLogSequenceNumber() > 0); - assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); - assertEquals(2, desc.getStoresCount()); - - StoreDescriptor store = desc.getStores(0); - assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); - assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); - assertEquals(1, store.getStoreFileCount()); // 1store file - assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative - - store = desc.getStores(1); - assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); - assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); - assertEquals(0, store.getStoreFileCount()); // no store files + region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), + TEST_UTIL.getConfiguration(), rss, null); - } finally { - HRegion.closeHRegion(region); - } + verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any() + , editCaptor.capture(), anyBoolean()); + + WALEdit edit = editCaptor.getValue(); + assertNotNull(edit); + assertNotNull(edit.getCells()); + assertEquals(1, edit.getCells().size()); + RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); + assertNotNull(desc); + + LOG.info("RegionEventDescriptor from WAL: " + desc); + + assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); + assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName())); + assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(), + hri.getEncodedNameAsBytes())); + assertTrue(desc.getLogSequenceNumber() > 0); + assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); + assertEquals(2, desc.getStoresCount()); + + StoreDescriptor store = desc.getStores(0); + assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); + assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); + assertEquals(1, store.getStoreFileCount()); // 1store file + assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative + + store = desc.getStores(1); + assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); + assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); + assertEquals(0, store.getStoreFileCount()); // no store files } // Helper for test testOpenRegionWrittenToWALForLogReplay @@ -6198,7 +5871,7 @@ public class TestHRegion { HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY); // open the region w/o rss and wal and flush some files - HRegion region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL + region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL .getConfiguration(), htd); assertNotNull(region); @@ -6218,56 +5891,51 @@ public class TestHRegion { recoveringRegions.put(region.getRegionInfo().getEncodedName(), null); when(rss.getRecoveringRegions()).thenReturn(recoveringRegions); - try { - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.set(HConstants.REGION_IMPL, HRegionWithSeqId.class.getName()); - region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), - conf, rss, null); - - // verify that we have not appended region open event to WAL because this region is still - // recovering - verify(wal, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any() - , editCaptor.capture(), anyBoolean()); - - // not put the region out of recovering state - new FinishRegionRecoveringHandler(rss, region.getRegionInfo().getEncodedName(), "/foo") - .prepare().process(); - - // now we should have put the entry - verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any() - , editCaptor.capture(), anyBoolean()); - - WALEdit edit = editCaptor.getValue(); - assertNotNull(edit); - assertNotNull(edit.getCells()); - assertEquals(1, edit.getCells().size()); - RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); - assertNotNull(desc); - - LOG.info("RegionEventDescriptor from WAL: " + desc); - - assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); - assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName())); - assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(), - hri.getEncodedNameAsBytes())); - assertTrue(desc.getLogSequenceNumber() > 0); - assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); - assertEquals(2, desc.getStoresCount()); - - StoreDescriptor store = desc.getStores(0); - assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); - assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); - assertEquals(1, store.getStoreFileCount()); // 1store file - assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative - - store = desc.getStores(1); - assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); - assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); - assertEquals(0, store.getStoreFileCount()); // no store files + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + conf.set(HConstants.REGION_IMPL, HRegionWithSeqId.class.getName()); + region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), + conf, rss, null); - } finally { - HRegion.closeHRegion(region); - } + // verify that we have not appended region open event to WAL because this region is still + // recovering + verify(wal, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any() + , editCaptor.capture(), anyBoolean()); + + // not put the region out of recovering state + new FinishRegionRecoveringHandler(rss, region.getRegionInfo().getEncodedName(), "/foo") + .prepare().process(); + + // now we should have put the entry + verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any() + , editCaptor.capture(), anyBoolean()); + + WALEdit edit = editCaptor.getValue(); + assertNotNull(edit); + assertNotNull(edit.getCells()); + assertEquals(1, edit.getCells().size()); + RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); + assertNotNull(desc); + + LOG.info("RegionEventDescriptor from WAL: " + desc); + + assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); + assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName())); + assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(), + hri.getEncodedNameAsBytes())); + assertTrue(desc.getLogSequenceNumber() > 0); + assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); + assertEquals(2, desc.getStoresCount()); + + StoreDescriptor store = desc.getStores(0); + assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); + assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); + assertEquals(1, store.getStoreFileCount()); // 1store file + assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative + + store = desc.getStores(1); + assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); + assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); + assertEquals(0, store.getStoreFileCount()); // no store files } /** @@ -6429,117 +6097,112 @@ public class TestHRegion { Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); - HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(), + region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY), TEST_UTIL.getDataTestDir(), conf, htd); assertNotNull(region); - try { - long now = EnvironmentEdgeManager.currentTime(); - // Add a cell that will expire in 5 seconds via cell TTL - region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, - HConstants.EMPTY_BYTE_ARRAY, new Tag[] { - // TTL tags specify ts in milliseconds - new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } ))); - // Add a cell that will expire after 10 seconds via family setting - region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); - // Add a cell that will expire in 15 seconds via cell TTL - region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1, - HConstants.EMPTY_BYTE_ARRAY, new Tag[] { - // TTL tags specify ts in milliseconds - new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } ))); - // Add a cell that will expire in 20 seconds via family setting - region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY)); - - // Flush so we are sure store scanning gets this right - region.flush(true); - - // A query at time T+0 should return all cells - Result r = region.get(new Get(row)); - assertNotNull(r.getValue(fam1, q1)); - assertNotNull(r.getValue(fam1, q2)); - assertNotNull(r.getValue(fam1, q3)); - assertNotNull(r.getValue(fam1, q4)); - - // Increment time to T+5 seconds - edge.incrementTime(5000); - - r = region.get(new Get(row)); - assertNull(r.getValue(fam1, q1)); - assertNotNull(r.getValue(fam1, q2)); - assertNotNull(r.getValue(fam1, q3)); - assertNotNull(r.getValue(fam1, q4)); - - // Increment time to T+10 seconds - edge.incrementTime(5000); - - r = region.get(new Get(row)); - assertNull(r.getValue(fam1, q1)); - assertNull(r.getValue(fam1, q2)); - assertNotNull(r.getValue(fam1, q3)); - assertNotNull(r.getValue(fam1, q4)); - - // Increment time to T+15 seconds - edge.incrementTime(5000); - - r = region.get(new Get(row)); - assertNull(r.getValue(fam1, q1)); - assertNull(r.getValue(fam1, q2)); - assertNull(r.getValue(fam1, q3)); - assertNotNull(r.getValue(fam1, q4)); - - // Increment time to T+20 seconds - edge.incrementTime(10000); - - r = region.get(new Get(row)); - assertNull(r.getValue(fam1, q1)); - assertNull(r.getValue(fam1, q2)); - assertNull(r.getValue(fam1, q3)); - assertNull(r.getValue(fam1, q4)); - - // Fun with disappearing increments - - // Start at 1 - region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L))); - r = region.get(new Get(row)); - byte[] val = r.getValue(fam1, q1); - assertNotNull(val); - assertEquals(Bytes.toLong(val), 1L); - - // Increment with a TTL of 5 seconds - Increment incr = new Increment(row).addColumn(fam1, q1, 1L); - incr.setTTL(5000); - region.increment(incr); // 2 - - // New value should be 2 - r = region.get(new Get(row)); - val = r.getValue(fam1, q1); - assertNotNull(val); - assertEquals(Bytes.toLong(val), 2L); - - // Increment time to T+25 seconds - edge.incrementTime(5000); - - // Value should be back to 1 - r = region.get(new Get(row)); - val = r.getValue(fam1, q1); - assertNotNull(val); - assertEquals(Bytes.toLong(val), 1L); - - // Increment time to T+30 seconds - edge.incrementTime(5000); - - // Original value written at T+20 should be gone now via family TTL - r = region.get(new Get(row)); - assertNull(r.getValue(fam1, q1)); + long now = EnvironmentEdgeManager.currentTime(); + // Add a cell that will expire in 5 seconds via cell TTL + region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, + HConstants.EMPTY_BYTE_ARRAY, new Tag[] { + // TTL tags specify ts in milliseconds + new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } ))); + // Add a cell that will expire after 10 seconds via family setting + region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); + // Add a cell that will expire in 15 seconds via cell TTL + region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1, + HConstants.EMPTY_BYTE_ARRAY, new Tag[] { + // TTL tags specify ts in milliseconds + new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } ))); + // Add a cell that will expire in 20 seconds via family setting + region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY)); + + // Flush so we are sure store scanning gets this right + region.flush(true); - } finally { - HRegion.closeHRegion(region); - } + // A query at time T+0 should return all cells + Result r = region.get(new Get(row)); + assertNotNull(r.getValue(fam1, q1)); + assertNotNull(r.getValue(fam1, q2)); + assertNotNull(r.getValue(fam1, q3)); + assertNotNull(r.getValue(fam1, q4)); + + // Increment time to T+5 seconds + edge.incrementTime(5000); + + r = region.get(new Get(row)); + assertNull(r.getValue(fam1, q1)); + assertNotNull(r.getValue(fam1, q2)); + assertNotNull(r.getValue(fam1, q3)); + assertNotNull(r.getValue(fam1, q4)); + + // Increment time to T+10 seconds + edge.incrementTime(5000); + + r = region.get(new Get(row)); + assertNull(r.getValue(fam1, q1)); + assertNull(r.getValue(fam1, q2)); + assertNotNull(r.getValue(fam1, q3)); + assertNotNull(r.getValue(fam1, q4)); + + // Increment time to T+15 seconds + edge.incrementTime(5000); + + r = region.get(new Get(row)); + assertNull(r.getValue(fam1, q1)); + assertNull(r.getValue(fam1, q2)); + assertNull(r.getValue(fam1, q3)); + assertNotNull(r.getValue(fam1, q4)); + + // Increment time to T+20 seconds + edge.incrementTime(10000); + + r = region.get(new Get(row)); + assertNull(r.getValue(fam1, q1)); + assertNull(r.getValue(fam1, q2)); + assertNull(r.getValue(fam1, q3)); + assertNull(r.getValue(fam1, q4)); + + // Fun with disappearing increments + + // Start at 1 + region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L))); + r = region.get(new Get(row)); + byte[] val = r.getValue(fam1, q1); + assertNotNull(val); + assertEquals(Bytes.toLong(val), 1L); + + // Increment with a TTL of 5 seconds + Increment incr = new Increment(row).addColumn(fam1, q1, 1L); + incr.setTTL(5000); + region.increment(incr); // 2 + + // New value should be 2 + r = region.get(new Get(row)); + val = r.getValue(fam1, q1); + assertNotNull(val); + assertEquals(Bytes.toLong(val), 2L); + + // Increment time to T+25 seconds + edge.incrementTime(5000); + + // Value should be back to 1 + r = region.get(new Get(row)); + val = r.getValue(fam1, q1); + assertNotNull(val); + assertEquals(Bytes.toLong(val), 1L); + + // Increment time to T+30 seconds + edge.incrementTime(5000); + + // Original value written at T+20 should be gone now via family TTL + r = region.get(new Get(row)); + assertNull(r.getValue(fam1, q1)); } @Test public void testIncrementTimestampsAreMonotonic() throws IOException { - HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1); + region = initHRegion(tableName, name.getMethodName(), CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); @@ -6564,7 +6227,7 @@ public class TestHRegion { @Test public void testAppendTimestampsAreMonotonic() throws IOException { - HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1); + region = initHRegion(tableName, name.getMethodName(), CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); @@ -6595,7 +6258,7 @@ public class TestHRegion { @Test public void testCheckAndMutateTimestampsAreMonotonic() throws IOException { - HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1); + region = initHRegion(tableName, name.getMethodName(), CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); @@ -6631,7 +6294,7 @@ public class TestHRegion { int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000); CONF.setInt("hbase.rowlock.wait.duration", 1000); - final HRegion region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1); + region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1); Mutation[] mutations = new Mutation[] { new Put(a).addImmutable(fam1, null, null), @@ -6687,7 +6350,7 @@ public class TestHRegion { @Test public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { - HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1); + region = initHRegion(tableName, name.getMethodName(), CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); @@ -6807,15 +6470,10 @@ public class TestHRegion { rm.add(put); this.region = initHRegion(tableName, method, CONF, fam1); - try { - long wrcBeforeMutate = this.region.writeRequestsCount.get(); - this.region.mutateRow(rm); - long wrcAfterMutate = this.region.writeRequestsCount.get(); - Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate); - } finally { - HBaseTestingUtility.closeRegionAndWAL(this.region); - this.region = null; - } + long wrcBeforeMutate = this.region.writeRequestsCount.get(); + this.region.mutateRow(rm); + long wrcAfterMutate = this.region.writeRequestsCount.get(); + Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate); } @Test @@ -6826,56 +6484,51 @@ public class TestHRegion { String method = this.getName(); HBaseConfiguration conf = new HBaseConfiguration(); this.region = initHRegion(tableName, method, conf, families); - try { - // setup with one storefile and one memstore, to create scanner and get an earlier readPt - Put put = new Put(Bytes.toBytes("19996")); - put.addColumn(cf1, col, Bytes.toBytes("val")); - region.put(put); - Put put2 = new Put(Bytes.toBytes("19995")); - put2.addColumn(cf1, col, Bytes.toBytes("val")); - region.put(put2); - - // create a reverse scan - Scan scan = new Scan(Bytes.toBytes("19996")); - scan.setReversed(true); - RegionScanner scanner = region.getScanner(scan); - - // flush the cache. This will reset the store scanner - region.flushcache(true, true); - - // create one memstore contains many rows will be skipped - // to check MemStoreScanner.seekToPreviousRow - for (int i = 10000; i < 20000; i++) { - Put p = new Put(Bytes.toBytes("" + i)); - p.addColumn(cf1, col, Bytes.toBytes("" + i)); - region.put(p); - } + // setup with one storefile and one memstore, to create scanner and get an earlier readPt + Put put = new Put(Bytes.toBytes("19996")); + put.addColumn(cf1, col, Bytes.toBytes("val")); + region.put(put); + Put put2 = new Put(Bytes.toBytes("19995")); + put2.addColumn(cf1, col, Bytes.toBytes("val")); + region.put(put2); + + // create a reverse scan + Scan scan = new Scan(Bytes.toBytes("19996")); + scan.setReversed(true); + RegionScanner scanner = region.getScanner(scan); + + // flush the cache. This will reset the store scanner + region.flushcache(true, true); + + // create one memstore contains many rows will be skipped + // to check MemStoreScanner.seekToPreviousRow + for (int i = 10000; i < 20000; i++) { + Put p = new Put(Bytes.toBytes("" + i)); + p.addColumn(cf1, col, Bytes.toBytes("" + i)); + region.put(p); + } - List currRow = new ArrayList<>(); - boolean hasNext; - boolean assertDone = false; - do { - hasNext = scanner.next(currRow); - // With HBASE-15871, after the scanner is reset the memstore scanner should not be - // added here - if (!assertDone) { - StoreScanner current = - (StoreScanner) (((RegionScannerImpl) scanner).storeHeap).getCurrentForTesting(); - List scanners = current.getAllScannersForTesting(); - assertEquals("There should be only one scanner the store file scanner", 1, - scanners.size()); - assertDone = true; - } - } while (hasNext); + List currRow = new ArrayList<>(); + boolean hasNext; + boolean assertDone = false; + do { + hasNext = scanner.next(currRow); + // With HBASE-15871, after the scanner is reset the memstore scanner should not be + // added here + if (!assertDone) { + StoreScanner current = + (StoreScanner) (((RegionScannerImpl) scanner).storeHeap).getCurrentForTesting(); + List scanners = current.getAllScannersForTesting(); + assertEquals("There should be only one scanner the store file scanner", 1, + scanners.size()); + assertDone = true; + } + } while (hasNext); - assertEquals(2, currRow.size()); - assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(), - currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); - assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(), - currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); - } finally { - HBaseTestingUtility.closeRegionAndWAL(this.region); - this.region = null; - } + assertEquals(2, currRow.size()); + assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(), + currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); + assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(), + currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); } } -- 2.18.0