diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 89ae0d1..bcde01c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -41,12 +41,10 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.TimeRange; -import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.CollectionBackedScanner; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.htrace.Trace; /** @@ -68,11 +66,6 @@ import org.apache.htrace.Trace; @InterfaceAudience.Private public class DefaultMemStore implements MemStore { private static final Log LOG = LogFactory.getLog(DefaultMemStore.class); - static final String USEMSLAB_KEY = "hbase.hregion.memstore.mslab.enabled"; - private static final boolean USEMSLAB_DEFAULT = true; - static final String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class"; - - private Configuration conf; // MemStore. Use a CellSkipListSet rather than SkipListSet because of the // better semantics. The Map will overwrite if passed a key it already had @@ -96,8 +89,6 @@ public class DefaultMemStore implements MemStore { TimeRangeTracker timeRangeTracker; TimeRangeTracker snapshotTimeRangeTracker; - volatile MemStoreLAB allocator; - volatile MemStoreLAB snapshotAllocator; volatile long snapshotId; volatile boolean tagsPresent; @@ -114,7 +105,6 @@ public class DefaultMemStore implements MemStore { */ public DefaultMemStore(final Configuration conf, final CellComparator c) { - this.conf = conf; this.comparator = c; this.cellSet = new CellSkipListSet(c); this.snapshot = new CellSkipListSet(c); @@ -122,13 +112,6 @@ public class DefaultMemStore implements MemStore { snapshotTimeRangeTracker = new TimeRangeTracker(); this.size = new AtomicLong(DEEP_OVERHEAD); this.snapshotSize = 0; - if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { - String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); - this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, - new Class[] { Configuration.class }, new Object[] { conf }); - } else { - this.allocator = null; - } } void dump() { @@ -161,13 +144,6 @@ public class DefaultMemStore implements MemStore { this.timeRangeTracker = new TimeRangeTracker(); // Reset heap to not include any keys this.size.set(DEEP_OVERHEAD); - this.snapshotAllocator = this.allocator; - // Reset allocator so we get a fresh buffer for the new memstore - if (allocator != null) { - String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); - this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, - new Class[] { Configuration.class }, new Object[] { conf }); - } timeOfOldestEdit = Long.MAX_VALUE; } } @@ -186,7 +162,6 @@ public class DefaultMemStore implements MemStore { */ @Override public void clearSnapshot(long id) throws UnexpectedStateException { - MemStoreLAB tmpAllocator = null; if (this.snapshotId != id) { throw new UnexpectedStateException("Current snapshot id is " + this.snapshotId + ",passed " + id); @@ -199,13 +174,6 @@ public class DefaultMemStore implements MemStore { } this.snapshotSize = 0; this.snapshotId = -1; - if (this.snapshotAllocator != null) { - tmpAllocator = this.snapshotAllocator; - this.snapshotAllocator = null; - } - if (tmpAllocator != null) { - tmpAllocator.close(); - } } @Override @@ -225,8 +193,7 @@ public class DefaultMemStore implements MemStore { */ @Override public long add(Cell cell) { - Cell toAdd = maybeCloneWithAllocator(cell); - return internalAdd(toAdd); + return internalAdd(cell); } @Override @@ -272,25 +239,6 @@ public class DefaultMemStore implements MemStore { return s; } - private Cell maybeCloneWithAllocator(Cell cell) { - if (allocator == null) { - return cell; - } - - int len = KeyValueUtil.length(cell); - ByteRange alloc = allocator.allocateBytes(len); - if (alloc == null) { - // The allocation was too large, allocator decided - // not to do anything with it. - return cell; - } - assert alloc.getBytes() != null; - KeyValueUtil.appendToByteArray(cell, alloc.getBytes(), alloc.getOffset()); - KeyValue newKv = new KeyValue(alloc.getBytes(), alloc.getOffset(), len); - newKv.setSequenceId(cell.getSequenceId()); - return newKv; - } - /** * Remove n key from the memstore. Only cells that have the same key and the * same memstoreTS are removed. It is ok to not update timeRangeTracker @@ -329,9 +277,8 @@ public class DefaultMemStore implements MemStore { @Override public long delete(Cell deleteCell) { long s = 0; - Cell toAdd = maybeCloneWithAllocator(deleteCell); - s += heapSizeChange(toAdd, addToCellSet(toAdd)); - timeRangeTracker.includeTimestamp(toAdd); + s += heapSizeChange(deleteCell, addToCellSet(deleteCell)); + timeRangeTracker.includeTimestamp(deleteCell); this.size.addAndGet(s); return s; } @@ -411,7 +358,7 @@ public class DefaultMemStore implements MemStore { if (!snSs.isEmpty()) { Cell snc = snSs.first(); // is there a matching Cell in the snapshot? - if (CellUtil.matchingRow(snc, firstCell) && CellUtil.matchingQualifier(snc, firstCell)) { + if (CellUtil.matchingRows(snc, firstCell) && CellUtil.matchingQualifier(snc, firstCell)) { if (snc.getTimestamp() == now) { // poop, now += 1; @@ -429,7 +376,7 @@ public class DefaultMemStore implements MemStore { for (Cell cell : ss) { // if this isnt the row we are interested in, then bail: if (!CellUtil.matchingColumn(cell, family, qualifier) - || !CellUtil.matchingRow(cell, firstCell)) { + || !CellUtil.matchingRows(cell, firstCell)) { break; // rows dont match, bail. } @@ -515,7 +462,7 @@ public class DefaultMemStore implements MemStore { continue; } // check that this is the row and column we are interested in, otherwise bail - if (CellUtil.matchingRow(cell, cur) && CellUtil.matchingQualifier(cell, cur)) { + if (CellUtil.matchingRows(cell, cur) && CellUtil.matchingQualifier(cell, cur)) { // only remove Puts that concurrent scanners cannot possibly see if (cur.getTypeByte() == KeyValue.Type.Put.getCode() && cur.getSequenceId() <= readpoint) { @@ -545,8 +492,8 @@ public class DefaultMemStore implements MemStore { * @return scanner on memstore and snapshot in this order. */ @Override - public List getScanners(long readPt) { - return Collections. singletonList(new MemStoreScanner(readPt)); + public List getScanners(long readPt, ScannerCloseCallback callback) { + return Collections. singletonList(new MemStoreScanner(readPt, callback)); } /** @@ -595,15 +542,12 @@ public class DefaultMemStore implements MemStore { // the pre-calculated Cell to be returned by peek() or next() private Cell theNext; - // The allocator and snapshot allocator at the time of creating this scanner - volatile MemStoreLAB allocatorAtCreation; - volatile MemStoreLAB snapshotAllocatorAtCreation; - // A flag represents whether could stop skipping Cells for MVCC // if have encountered the next row. Only used for reversed scan private boolean stopSkippingCellsIfNextRow = false; private long readPoint; + private ScannerCloseCallback callback; /* Some notes... @@ -626,20 +570,13 @@ public class DefaultMemStore implements MemStore { the adds to kvset in the MemStoreScanner. */ - MemStoreScanner(long readPoint) { + MemStoreScanner(long readPoint, ScannerCloseCallback callback) { super(); this.readPoint = readPoint; + this.callback = callback; cellSetAtCreation = cellSet; snapshotAtCreation = snapshot; - if (allocator != null) { - this.allocatorAtCreation = allocator; - this.allocatorAtCreation.incScannerCount(); - } - if (snapshotAllocator != null) { - this.snapshotAllocatorAtCreation = snapshotAllocator; - this.snapshotAllocatorAtCreation.incScannerCount(); - } if (Trace.isTracing() && Trace.currentSpan() != null) { Trace.currentSpan().addTimelineAnnotation("Creating MemStoreScanner"); } @@ -812,17 +749,11 @@ public class DefaultMemStore implements MemStore { this.cellSetIt = null; this.snapshotIt = null; - if (allocatorAtCreation != null) { - this.allocatorAtCreation.decScannerCount(); - this.allocatorAtCreation = null; - } - if (snapshotAllocatorAtCreation != null) { - this.snapshotAllocatorAtCreation.decScannerCount(); - this.snapshotAllocatorAtCreation = null; - } - this.cellSetItRow = null; this.snapshotItRow = null; + if (this.callback != null) { + this.callback.closed(); + } } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 9ebdaee..9ef3224 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; @@ -83,6 +84,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.security.EncryptionUtil; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.ClassSize; @@ -119,7 +121,9 @@ public class HStore implements Store { public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles"; public static final int DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER = 1000; public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7; - + public static final String USEMSLAB_KEY = "hbase.hregion.memstore.mslab.enabled"; + public static final boolean USEMSLAB_DEFAULT = true; + public static final String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class"; private static final Log LOG = LogFactory.getLog(HStore.class); protected final MemStore memstore; @@ -187,6 +191,10 @@ public class HStore implements Store { private volatile long compactedCellsSize = 0; private volatile long majorCompactedCellsSize = 0; + private boolean useMSLAB; + private volatile MemStoreLAB allocator; + private volatile MemStoreLAB snapshotAllocator; + /** * Constructor * @param region @@ -269,6 +277,14 @@ public class HStore implements Store { + flushRetriesNumber); } cryptoContext = EncryptionUtil.createEncryptionContext(conf, family); + this.useMSLAB = conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT); + if (this.useMSLAB) { + String mslabClassName = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); + this.allocator = ReflectionUtils.instantiateWithCustomCtor(mslabClassName, + new Class[] { Configuration.class }, new Object[] { conf }); + } else { + this.allocator = null; + } } /** @@ -619,32 +635,36 @@ public class HStore implements Store { public long add(final Cell cell) { lock.readLock().lock(); try { - return this.memstore.add(cell); + return this.memstore.add(maybeCloneWithAllocator(cell)); } finally { lock.readLock().unlock(); } } + private Cell maybeCloneWithAllocator(Cell cell) { + if (allocator == null) { + return cell; + } + + int len = KeyValueUtil.length(cell); + ByteRange alloc = allocator.allocateBytes(len); + if (alloc == null) { + // The allocation was too large, allocator decided + // not to do anything with it. + return cell; + } + assert alloc.getBytes() != null; + KeyValueUtil.appendToByteArray(cell, alloc.getBytes(), alloc.getOffset()); + KeyValue newKv = new KeyValue(alloc.getBytes(), alloc.getOffset(), len); + newKv.setSequenceId(cell.getSequenceId()); + return newKv; + } + @Override public long timeOfOldestEdit() { return memstore.timeOfOldestEdit(); } - /** - * Adds a value to the memstore - * - * @param kv - * @return memstore size delta - */ - protected long delete(final KeyValue kv) { - lock.readLock().lock(); - try { - return this.memstore.delete(kv); - } finally { - lock.readLock().unlock(); - } - } - @Override public void rollback(final Cell cell) { lock.readLock().lock(); @@ -1022,6 +1042,14 @@ public class HStore implements Store { this.storeEngine.getStoreFileManager().insertNewFiles(sfs); if (snapshotId > 0) { this.memstore.clearSnapshot(snapshotId); + MemStoreLAB tmpAllocator = null; + if (this.snapshotAllocator != null) { + tmpAllocator = this.snapshotAllocator; + this.snapshotAllocator = null; + } + if (tmpAllocator != null) { + tmpAllocator.close(); + } } } finally { // We need the lock, as long as we are updating the storeFiles @@ -1071,7 +1099,9 @@ public class HStore implements Store { try { storeFilesToScan = this.storeEngine.getStoreFileManager().getFilesForScanOrGet(isGet, startRow, stopRow); - memStoreScanners = this.memstore.getScanners(readPt); + memStoreScanners = this.memstore.getScanners(readPt, + new MemstoreScannerClosedCallback(this.allocator, this.snapshotAllocator)); + incMSLABScannerCount(); } finally { this.lock.readLock().unlock(); } @@ -1099,7 +1129,9 @@ public class HStore implements Store { if (includeMemstoreScanner) { this.lock.readLock().lock(); try { - memStoreScanners = this.memstore.getScanners(readPt); + memStoreScanners = this.memstore.getScanners(readPt, + new MemstoreScannerClosedCallback(this.allocator, this.snapshotAllocator)); + incMSLABScannerCount(); } finally { this.lock.readLock().unlock(); } @@ -1115,6 +1147,38 @@ public class HStore implements Store { return scanners; } + private void incMSLABScannerCount() { + if (this.allocator != null) { + this.allocator.incScannerCount(); + } + if (this.snapshotAllocator != null) { + this.snapshotAllocator.incScannerCount(); + } + } + + private final class MemstoreScannerClosedCallback implements ScannerCloseCallback { + + private MemStoreLAB allocator; + private MemStoreLAB snapshotAllocator; + + private MemstoreScannerClosedCallback(MemStoreLAB allocator, MemStoreLAB snapshotAllocator) { + this.allocator = allocator; + this.snapshotAllocator = snapshotAllocator; + } + + @Override + public void closed() { + if (this.allocator != null) { + this.allocator.decScannerCount(); + this.allocator = null; + } + if (this.snapshotAllocator != null) { + this.snapshotAllocator.decScannerCount(); + this.snapshotAllocator = null; + } + } + } + @Override public void addChangedReaderObserver(ChangedReadersObserver o) { this.changedReaderObservers.add(o); @@ -2051,6 +2115,18 @@ public class HStore implements Store { @Override public void prepare() { this.snapshot = memstore.snapshot(); + if (useMSLAB) { + // When MSLAB in use snapshotAllocator will be not null when the above snapshot call + // resulted in not actually taking a new snapshot(As an older snapshot flush was not yet + // over) + if (snapshotAllocator == null) { + snapshotAllocator = allocator; + // Reset allocator so we get a fresh buffer for the new memstore + String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); + allocator = ReflectionUtils.instantiateWithCustomCtor(className, + new Class[] { Configuration.class }, new Object[] { conf }); + } + } this.cacheFlushCount = snapshot.getCellsCount(); this.cacheFlushSize = snapshot.getSize(); committedFiles = new ArrayList(1); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java index e9f8103..6590c24 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java @@ -128,7 +128,7 @@ public interface MemStore extends HeapSize { * @return scanner over the memstore. This might include scanner over the snapshot when one is * present. */ - List getScanners(long readPt); + List getScanners(long readPt, ScannerCloseCallback callback); /** * @return Total memory occupied by this MemStore. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerCloseCallback.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerCloseCallback.java new file mode 100644 index 0000000..3e1fa0b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerCloseCallback.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Provides a callback mechanism so that other parts can get notified when a {@link KeyValueScanner} + * is getting closed. The scanner is supposed to call {@link #closed()} method once it is closed + */ +@InterfaceAudience.Private +public interface ScannerCloseCallback { + + /** + * Will be called when a scanner is getting closed + */ + void closed(); +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index ec70740..02f879b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -102,7 +102,7 @@ public class TestDefaultMemStore extends TestCase { */ public void testScanAcrossSnapshot() throws IOException { int rowCount = addRows(this.memstore); - List memstorescanners = this.memstore.getScanners(0); + List memstorescanners = this.memstore.getScanners(0, null); Scan scan = new Scan(); List result = new ArrayList(); Configuration conf = HBaseConfiguration.create(); @@ -128,7 +128,7 @@ public class TestDefaultMemStore extends TestCase { scanner.close(); } - memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); + memstorescanners = this.memstore.getScanners(mvcc.getReadPoint(), null); // Now assert can count same number even if a snapshot mid-scan. s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); count = 0; @@ -153,7 +153,7 @@ public class TestDefaultMemStore extends TestCase { for (KeyValueScanner scanner : memstorescanners) { scanner.close(); } - memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); + memstorescanners = this.memstore.getScanners(mvcc.getReadPoint(), null); // Assert that new values are seen in kvset as we scan. long ts = System.currentTimeMillis(); s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); @@ -218,7 +218,7 @@ public class TestDefaultMemStore extends TestCase { private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) throws IOException { - List memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); + List memstorescanners = this.memstore.getScanners(mvcc.getReadPoint(), null); assertEquals(1, memstorescanners.size()); final KeyValueScanner scanner = memstorescanners.get(0); scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW)); @@ -259,12 +259,12 @@ public class TestDefaultMemStore extends TestCase { kv1.setSequenceId(w.getWriteNumber()); memstore.add(kv1); - KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{}); mvcc.completeAndWait(w); - s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv1}); w = mvcc.begin(); @@ -272,12 +272,12 @@ public class TestDefaultMemStore extends TestCase { kv2.setSequenceId(w.getWriteNumber()); memstore.add(kv2); - s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv1}); mvcc.completeAndWait(w); - s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv1, kv2}); } @@ -309,7 +309,7 @@ public class TestDefaultMemStore extends TestCase { mvcc.completeAndWait(w); // BEFORE STARTING INSERT 2, SEE FIRST KVS - KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv11, kv12}); // START INSERT 2: Write both columns val2 @@ -323,7 +323,7 @@ public class TestDefaultMemStore extends TestCase { memstore.add(kv22); // BEFORE COMPLETING INSERT 2, SEE FIRST KVS - s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv11, kv12}); // COMPLETE INSERT 2 @@ -332,7 +332,7 @@ public class TestDefaultMemStore extends TestCase { // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS. // See HBASE-1485 for discussion about what we should do with // the duplicate-TS inserts - s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12}); } @@ -361,7 +361,7 @@ public class TestDefaultMemStore extends TestCase { mvcc.completeAndWait(w); // BEFORE STARTING INSERT 2, SEE FIRST KVS - KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv11, kv12}); // START DELETE: Insert delete for one of the columns @@ -372,14 +372,14 @@ public class TestDefaultMemStore extends TestCase { memstore.add(kvDel); // BEFORE COMPLETING DELETE, SEE FIRST KVS - s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv11, kv12}); // COMPLETE DELETE mvcc.completeAndWait(w); // NOW WE SHOULD SEE DELETE - s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12}); } @@ -434,7 +434,7 @@ public class TestDefaultMemStore extends TestCase { mvcc.completeAndWait(w); // Assert that we can read back - KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint(), null).get(0); s.seek(kv); Cell ret = s.next(); @@ -533,7 +533,7 @@ public class TestDefaultMemStore extends TestCase { ScanType scanType = ScanType.USER_SCAN; InternalScanner scanner = new StoreScanner(new Scan( Bytes.toBytes(startRowId)), scanInfo, scanType, null, - memstore.getScanners(0)); + memstore.getScanners(0, null)); List results = new ArrayList(); for (int i = 0; scanner.next(results); i++) { int rowId = startRowId + i; @@ -795,7 +795,7 @@ public class TestDefaultMemStore extends TestCase { */ public void testUpsertMSLAB() throws Exception { Configuration conf = HBaseConfiguration.create(); - conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true); + conf.setBoolean(HStore.USEMSLAB_KEY, true); memstore = new DefaultMemStore(conf, CellComparator.COMPARATOR); int ROW_SIZE = 2048; @@ -1067,7 +1067,7 @@ public class TestDefaultMemStore extends TestCase { static void doScan(MemStore ms, int iteration) throws IOException { long nanos = System.nanoTime(); - KeyValueScanner s = ms.getScanners(0).get(0); + KeyValueScanner s = ms.getScanners(0, null).get(0); s.seek(KeyValueUtil.createFirstOnRow(new byte[]{})); System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java index 80333e8..f83c043 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java @@ -47,7 +47,7 @@ public class TestMemStoreChunkPool { @BeforeClass public static void setUpBeforeClass() throws Exception { - conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true); + conf.setBoolean(HStore.USEMSLAB_KEY, true); conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f); chunkPoolDisabledBeforeTest = MemStoreChunkPool.chunkPoolDisabled; MemStoreChunkPool.chunkPoolDisabled = false; @@ -162,7 +162,7 @@ public class TestMemStoreChunkPool { assertEquals(2, memstore.cellSet.size()); // opening scanner before clear the snapshot - List scanners = memstore.getScanners(0); + List scanners = memstore.getScanners(0, null); // Shouldn't putting back the chunks to pool,since some scanners are opening // based on their data memstore.clearSnapshot(snapshot.getId()); @@ -184,7 +184,7 @@ public class TestMemStoreChunkPool { memstore.add(new KeyValue(row, fam, qf6, val)); memstore.add(new KeyValue(row, fam, qf7, val)); // opening scanners - scanners = memstore.getScanners(0); + scanners = memstore.getScanners(0, null); // close scanners before clear the snapshot for (KeyValueScanner scanner : scanners) { scanner.close(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java index c6cc902..3301563 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java @@ -124,11 +124,11 @@ public class TestReversibleScanners { public void testReversibleMemstoreScanner() throws IOException { MemStore memstore = new DefaultMemStore(); writeMemstore(memstore); - List scanners = memstore.getScanners(Long.MAX_VALUE); + List scanners = memstore.getScanners(Long.MAX_VALUE, null); seekTestOfReversibleKeyValueScanner(scanners.get(0)); for (int readPoint = 0; readPoint < MAXMVCC; readPoint++) { LOG.info("Setting read point to " + readPoint); - scanners = memstore.getScanners(readPoint); + scanners = memstore.getScanners(readPoint, null); seekTestOfReversibleKeyValueScannerWithMVCC(scanners.get(0), readPoint); } @@ -495,7 +495,7 @@ public class TestReversibleScanners { List fileScanners = StoreFileScanner .getScannersForStoreFiles(Lists.newArrayList(sf1, sf2), false, true, false, false, readPoint); - List memScanners = memstore.getScanners(readPoint); + List memScanners = memstore.getScanners(readPoint, null); List scanners = new ArrayList( fileScanners.size() + 1); scanners.addAll(fileScanners);