diff --git src/java/org/apache/hadoop/hbase/regionserver/HRegion.java src/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 1098ce3..cc5af5d 100644 --- src/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ src/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1823,10 +1823,9 @@ public class HRegion implements HConstants, HeapSize { // , Writable{ private Filter filter; private RowFilterInterface oldFilter; private List results = new ArrayList(); - private Scan theScan = null; private int isScan; - private List extraScanners = null; private boolean filterClosed = false; + private long readPt; RegionScanner(Scan scan, List additionalScanners) { //DebugPrint.println("HRegionScanner."); @@ -1840,29 +1839,26 @@ public class HRegion implements HConstants, HeapSize { // , Writable{ this.stopRow = scan.getStopRow(); } this.isScan = scan.isGetScan() ? -1 : 0; - this.theScan = scan; - this.extraScanners = additionalScanners; - } - RegionScanner(Scan scan) { - this(scan, null); - } + this.readPt = ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); - void initHeap() { List scanners = new ArrayList(); - if (extraScanners != null) { - scanners.addAll(extraScanners); + if (additionalScanners != null) { + scanners.addAll(additionalScanners); } for (Map.Entry> entry : - theScan.getFamilyMap().entrySet()) { + scan.getFamilyMap().entrySet()) { Store store = stores.get(entry.getKey()); - scanners.add(store.getScanner(theScan, entry.getValue())); + scanners.add(store.getScanner(scan, entry.getValue())); } this.storeHeap = new KeyValueHeap(scanners.toArray(new KeyValueScanner[0]), comparator); } + RegionScanner(Scan scan) { + this(scan, null); + } /** * Reset both the filter and the old filter. @@ -1876,7 +1872,7 @@ public class HRegion implements HConstants, HeapSize { // , Writable{ } // Start the next row read and reset the thread point - ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); + this.readPt = ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); } public synchronized boolean next(List outResults) @@ -1893,11 +1889,7 @@ public class HRegion implements HConstants, HeapSize { // , Writable{ } // This could be a new thread from the last time we called next(). - ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); - // lazy init the store heap. - if (storeHeap == null) { - initHeap(); - } + ReadWriteConsistencyControl.setThreadReadPoint(this.readPt); results.clear(); boolean returnResult = nextInternal(); diff --git src/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java src/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java index 3ec91a8..da2c9e0 100644 --- src/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java +++ src/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java @@ -39,7 +39,11 @@ public class ReadWriteConsistencyControl { public static long getThreadReadPoint() { return perThreadReadPoint.get(); } - + + public static void setThreadReadPoint(long readPoint) { + perThreadReadPoint.set(readPoint); + } + public static long resetThreadReadPoint(ReadWriteConsistencyControl rwcc) { perThreadReadPoint.set(rwcc.memstoreReadPoint()); return getThreadReadPoint(); diff --git src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java index fa9af03..5e73db3 100644 --- src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -58,7 +58,7 @@ import org.apache.hadoop.hbase.util.Bytes; /** * Basic stand-alone testing of HRegion. - * + * * A lot of the meta information for an HRegion now lives inside other * HRegions or in the HBaseMaster, so only basic testing is possible. */ @@ -67,7 +67,7 @@ public class TestHRegion extends HBaseTestCase { HRegion region = null; private final String DIR = "test/build/data/TestHRegion/"; - + private final int MAX_VERSIONS = 2; // Test names @@ -88,7 +88,7 @@ public class TestHRegion extends HBaseTestCase { } ////////////////////////////////////////////////////////////////////////////// - // New tests that doesn't spin up a mini cluster but rather just test the + // New tests that doesn't spin up a mini cluster but rather just test the // individual code pieces in the HRegion. Putting files locally in // /tmp/testtable ////////////////////////////////////////////////////////////////////////////// @@ -97,7 +97,7 @@ public class TestHRegion extends HBaseTestCase { HBaseConfiguration hc = initSplit(); int numRows = 100; byte [][] families = {fam1, fam2, fam3}; - + //Setting up region String method = this.getName(); initHRegion(tableName, method, hc, families); @@ -333,7 +333,7 @@ public class TestHRegion extends HBaseTestCase { byte [] val1 = Bytes.toBytes("value1"); byte [] val2 = Bytes.toBytes("value2"); Integer lockId = null; - + //Setting up region String method = this.getName(); initHRegion(tableName, method, fam1); @@ -356,7 +356,7 @@ public class TestHRegion extends HBaseTestCase { res = region.checkAndPut(row1, fam1, qf1, val1, put, lockId, true); assertTrue(res); } - + public void testCheckAndPut_WithWrongValue() throws IOException{ byte [] tableName = Bytes.toBytes("testtable"); byte [] row1 = Bytes.toBytes("row1"); @@ -374,7 +374,7 @@ public class TestHRegion extends HBaseTestCase { Put put = new Put(row1); put.add(fam1, qf1, val1); region.put(put); - + //checkAndPut with wrong value boolean res = region.checkAndPut(row1, fam1, qf1, val2, put, lockId, true); assertEquals(false, res); @@ -396,12 +396,12 @@ public class TestHRegion extends HBaseTestCase { Put put = new Put(row1); put.add(fam1, qf1, val1); region.put(put); - + //checkAndPut with correct value boolean res = region.checkAndPut(row1, fam1, qf1, val1, put, lockId, true); assertEquals(true, res); } - + public void testCheckAndPut_ThatPutWasWritten() throws IOException{ byte [] tableName = Bytes.toBytes("testtable"); byte [] row1 = Bytes.toBytes("row1"); @@ -413,7 +413,7 @@ public class TestHRegion extends HBaseTestCase { Integer lockId = null; byte [][] families = {fam1, fam2}; - + //Setting up region String method = this.getName(); initHRegion(tableName, method, families); @@ -422,34 +422,34 @@ public class TestHRegion extends HBaseTestCase { Put put = new Put(row1); put.add(fam1, qf1, val1); region.put(put); - + //Creating put to add long ts = System.currentTimeMillis(); KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); put = new Put(row1); put.add(kv); - + //checkAndPut with wrong value Store store = region.getStore(fam1); store.memstore.kvset.size(); - + boolean res = region.checkAndPut(row1, fam1, qf1, val1, put, lockId, true); assertEquals(true, res); store.memstore.kvset.size(); - + Get get = new Get(row1); get.addColumn(fam2, qf1); KeyValue [] actual = region.get(get, null).raw(); - + KeyValue [] expected = {kv}; - + assertEquals(expected.length, actual.length); for(int i=0; i kvs = new ArrayList(); kvs.add(new KeyValue(row1, fam1, col1, null)); kvs.add(new KeyValue(row1, fam1, col2, null)); @@ -722,7 +722,7 @@ public class TestHRegion extends HBaseTestCase { now = kv.getTimestamp(); } } - + ////////////////////////////////////////////////////////////////////////////// // Get tests ////////////////////////////////////////////////////////////////////////////// @@ -732,14 +732,14 @@ public class TestHRegion extends HBaseTestCase { byte [] fam1 = Bytes.toBytes("fam1"); byte [] fam2 = Bytes.toBytes("False"); byte [] col1 = Bytes.toBytes("col1"); - + //Setting up region String method = this.getName(); initHRegion(tableName, method, fam1); - + Get get = new Get(row1); get.addColumn(fam2, col1); - + //Test try { region.get(get, null); @@ -759,11 +759,11 @@ public class TestHRegion extends HBaseTestCase { byte [] col3 = Bytes.toBytes("col3"); byte [] col4 = Bytes.toBytes("col4"); byte [] col5 = Bytes.toBytes("col5"); - + //Setting up region String method = this.getName(); initHRegion(tableName, method, fam1); - + //Add to memstore Put put = new Put(row1); put.add(fam1, col1, null); @@ -806,17 +806,17 @@ public class TestHRegion extends HBaseTestCase { byte [] tableName = Bytes.toBytes("emptytable"); byte [] row = Bytes.toBytes("row"); byte [] fam = Bytes.toBytes("fam"); - + String method = this.getName(); initHRegion(tableName, method, fam); - + Get get = new Get(row); get.addFamily(fam); Result r = region.get(get, null); - + assertTrue(r.isEmpty()); } - + //Test that checked if there was anything special when reading from the ROOT //table. To be able to use this test you need to comment the part in //HTableDescriptor that checks for '-' and '.'. You also need to remove the @@ -830,7 +830,7 @@ public class TestHRegion extends HBaseTestCase { Put put = new Put(HConstants.EMPTY_START_ROW); put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null); region.put(put); - + Get get = new Get(HConstants.EMPTY_START_ROW); get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); @@ -838,10 +838,10 @@ public class TestHRegion extends HBaseTestCase { KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); KeyValue [] expected = {kv1}; - + //Test from memstore Result res = region.get(get, null); - + assertEquals(expected.length, res.size()); for(int i=0; i result = new ArrayList(); s.next(result); - + assertEquals(expected.length, result.size()); for(int i=0; ithreads = new ArrayList(threadCount); for (int i = 0; i < threadCount; i++) { threads.add(new Thread(Integer.toString(i)) { @@ -923,7 +923,7 @@ public class TestHRegion extends HBaseTestCase { } LOG.debug(getName() + " set " + Integer.toString(lockCount) + " locks"); - + // Abort outstanding locks. for (int i = lockCount - 1; i >= 0; i--) { region.releaseRowLock(lockids[i]); @@ -934,12 +934,12 @@ public class TestHRegion extends HBaseTestCase { } }); } - + // Startup all our threads. for (Thread t : threads) { t.start(); } - + // Now wait around till all are done. for (Thread t: threads) { while (t.isAlive()) { @@ -952,19 +952,19 @@ public class TestHRegion extends HBaseTestCase { } LOG.info("locks completed."); } - + ////////////////////////////////////////////////////////////////////////////// // Merge test - ////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////// public void testMerge() throws IOException { byte [] tableName = Bytes.toBytes("testtable"); byte [][] families = {fam1, fam2, fam3}; - + HBaseConfiguration hc = initSplit(); //Setting up region String method = this.getName(); initHRegion(tableName, method, hc, families); - + try { LOG.info("" + addContent(region, fam3)); region.flushcache(); @@ -1019,7 +1019,7 @@ public class TestHRegion extends HBaseTestCase { } } } - + ////////////////////////////////////////////////////////////////////////////// // Scanner tests ////////////////////////////////////////////////////////////////////////////// @@ -1027,13 +1027,13 @@ public class TestHRegion extends HBaseTestCase { byte [] tableName = Bytes.toBytes("testtable"); byte [] fam1 = Bytes.toBytes("fam1"); byte [] fam2 = Bytes.toBytes("fam2"); - + byte [][] families = {fam1, fam2}; - + //Setting up region String method = this.getName(); initHRegion(tableName, method, families); - + Scan scan = new Scan(); scan.addFamily(fam1); scan.addFamily(fam2); @@ -1043,7 +1043,7 @@ public class TestHRegion extends HBaseTestCase { assertTrue("Families could not be found in Region", false); } } - + public void testGetScanner_WithNotOkFamilies() throws IOException { byte [] tableName = Bytes.toBytes("testtable"); byte [] fam1 = Bytes.toBytes("fam1"); @@ -1065,7 +1065,7 @@ public class TestHRegion extends HBaseTestCase { } assertTrue("Families could not be found in Region", ok); } - + public void testGetScanner_WithNoFamilies() throws IOException { byte [] tableName = Bytes.toBytes("testtable"); byte [] row1 = Bytes.toBytes("row1"); @@ -1073,13 +1073,13 @@ public class TestHRegion extends HBaseTestCase { byte [] fam2 = Bytes.toBytes("fam2"); byte [] fam3 = Bytes.toBytes("fam3"); byte [] fam4 = Bytes.toBytes("fam4"); - + byte [][] families = {fam1, fam2, fam3, fam4}; - + //Setting up region String method = this.getName(); initHRegion(tableName, method, families); - + //Putting data in Region Put put = new Put(row1); @@ -1088,23 +1088,21 @@ public class TestHRegion extends HBaseTestCase { put.add(fam3, null, null); put.add(fam4, null, null); region.put(put); - + Scan scan = null; HRegion.RegionScanner is = null; - - //Testing to see how many scanners that is produced by getScanner, starting + + //Testing to see how many scanners that is produced by getScanner, starting //with known number, 2 - current = 1 scan = new Scan(); scan.addFamily(fam2); scan.addFamily(fam4); is = (RegionScanner) region.getScanner(scan); - is.initHeap(); // i dont like this test assertEquals(1, ((RegionScanner)is).storeHeap.getHeap().size()); - + scan = new Scan(); is = (RegionScanner) region.getScanner(scan); - is.initHeap(); - assertEquals(families.length -1, + assertEquals(families.length -1, ((RegionScanner)is).storeHeap.getHeap().size()); } @@ -1116,14 +1114,14 @@ public class TestHRegion extends HBaseTestCase { byte [] fam2 = Bytes.toBytes("fam2"); byte [] fam3 = Bytes.toBytes("fam3"); byte [] fam4 = Bytes.toBytes("fam4"); - + byte [][] families = {fam1, fam2, fam3, fam4}; long ts = System.currentTimeMillis(); - + //Setting up region String method = this.getName(); initHRegion(tableName, method, families); - + //Putting data in Region Put put = null; put = new Put(row1); @@ -1139,39 +1137,39 @@ public class TestHRegion extends HBaseTestCase { put.add(fam3, null, ts, null); put.add(fam4, null, ts, null); region.put(put); - + Scan scan = new Scan(); scan.addFamily(fam2); scan.addFamily(fam4); InternalScanner is = region.getScanner(scan); - + List res = null; - + //Result 1 List expected1 = new ArrayList(); expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); - + res = new ArrayList(); is.next(res); for(int i=0; i expected2 = new ArrayList(); expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); - + res = new ArrayList(); is.next(res); for(int i=0; i expected = new ArrayList(); expected.add(kv13); expected.add(kv12); - + Scan scan = new Scan(row1); scan.addColumn(fam1, qf1); scan.setMaxVersions(MAX_VERSIONS); List actual = new ArrayList(); InternalScanner scanner = region.getScanner(scan); - + boolean hasNext = scanner.next(actual); assertEquals(false, hasNext); - + //Verify result for(int i=0; i expected = new ArrayList(); expected.add(kv13); expected.add(kv12); expected.add(kv23); expected.add(kv22); - + Scan scan = new Scan(row1); scan.addColumn(fam1, qf1); scan.addColumn(fam1, qf2); scan.setMaxVersions(MAX_VERSIONS); List actual = new ArrayList(); InternalScanner scanner = region.getScanner(scan); - + boolean hasNext = scanner.next(actual); assertEquals(false, hasNext); - + //Verify result for(int i=0; i expected = new ArrayList(); expected.add(kv14); @@ -1348,7 +1346,7 @@ public class TestHRegion extends HBaseTestCase { expected.add(kv24); expected.add(kv23); expected.add(kv22); - + Scan scan = new Scan(row1); scan.addColumn(fam1, qf1); scan.addColumn(fam1, qf2); @@ -1356,17 +1354,17 @@ public class TestHRegion extends HBaseTestCase { scan.setMaxVersions(versions); List actual = new ArrayList(); InternalScanner scanner = region.getScanner(scan); - + boolean hasNext = scanner.next(actual); assertEquals(false, hasNext); - + //Verify result for(int i=0; i expected = new ArrayList(); expected.add(kv13); expected.add(kv12); expected.add(kv23); expected.add(kv22); - + Scan scan = new Scan(row1); scan.addFamily(fam1); scan.setMaxVersions(MAX_VERSIONS); List actual = new ArrayList(); InternalScanner scanner = region.getScanner(scan); - + boolean hasNext = scanner.next(actual); assertEquals(false, hasNext); - + //Verify result for(int i=0; i expected = new ArrayList(); expected.add(kv13); expected.add(kv12); expected.add(kv23); expected.add(kv22); - + Scan scan = new Scan(row1); scan.addFamily(fam1); scan.setMaxVersions(MAX_VERSIONS); List actual = new ArrayList(); InternalScanner scanner = region.getScanner(scan); - + boolean hasNext = scanner.next(actual); assertEquals(false, hasNext); - + //Verify result for(int i=0; i expected = new ArrayList(); expected.add(kv14); @@ -1781,16 +1779,16 @@ public class TestHRegion extends HBaseTestCase { expected.add(kv24); expected.add(kv23); expected.add(kv22); - + Scan scan = new Scan(row1); int versions = 3; scan.setMaxVersions(versions); List actual = new ArrayList(); InternalScanner scanner = region.getScanner(scan); - + boolean hasNext = scanner.next(actual); assertEquals(false, hasNext); - + //Verify result for(int i=0; ifirstValue. * @param r @@ -2398,7 +2402,7 @@ public class TestHRegion extends HBaseTestCase { s.close(); } } - + protected HRegion [] split(final HRegion r, final byte [] splitRow) throws IOException { // Assert can get mid key from passed region. @@ -2407,18 +2411,18 @@ public class TestHRegion extends HBaseTestCase { assertEquals(regions.length, 2); return regions; } - + private HBaseConfiguration initSplit() { HBaseConfiguration conf = new HBaseConfiguration(); // Always compact if there is more than one store file. conf.setInt("hbase.hstore.compactionThreshold", 2); - + // Make lease timeout longer, lease checks less frequent conf.setInt("hbase.master.lease.period", 10 * 1000); conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000); - + conf.setInt("hbase.regionserver.lease.period", 10 * 1000); - + // Increase the amount of time between client retries conf.setLong("hbase.client.pause", 15 * 1000); @@ -2426,7 +2430,7 @@ public class TestHRegion extends HBaseTestCase { // below. After adding all data, the first region is 1.3M conf.setLong("hbase.hregion.max.filesize", 1024 * 128); return conf; - } + } private void initHRegion (byte [] tableName, String callingMethod, byte[] ... families)