From fc37fd8199dfbbe4329eb6f5a36588abc37a35cb Mon Sep 17 00:00:00 2001 From: dvdreddy Date: Wed, 13 Jan 2016 14:10:41 -0800 Subject: [PATCH] HBASE-15101 close un-selected KeyValueScanners to avoid leaking ref counts --- .../hadoop/hbase/regionserver/StoreScanner.java | 46 +++++++++++++++++----- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 3049608..c3183ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.util.Pair; /** * Scanner scans both the memstore and the Store. Coalesce KeyValue stream @@ -270,7 +271,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner } // Filter the list of scanners using Bloom filters, time range, TTL, etc. - scanners = selectScannersFrom(scanners); + scanners = selectScannersFromAndCloseUnSelected(scanners); // Seek all scanners to the initial key seekScanners(scanners, matcher.getStartKey(), false, parallelSeekEnabled); @@ -324,8 +325,8 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner protected List getScannersNoCompaction() throws IOException { final boolean isCompaction = false; boolean usePread = get || scanUsePread; - return selectScannersFrom(store.getScanners(cacheBlocks, get, usePread, - isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt)); + return selectScannersFromAndCloseUnSelected(store.getScanners(cacheBlocks, get, usePread, + isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt)); } /** @@ -376,8 +377,11 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner /** * Filters the given list of scanners using Bloom filter, time range, and * TTL. + * Returns a pair of lists of scanners where the first element is the + * selected scanners and the second element is the ignored scanners */ - protected List selectScannersFrom( + protected Pair, List> + selectScannersFrom( final List allScanners) { boolean memOnly; boolean filesOnly; @@ -390,8 +394,11 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner filesOnly = false; } - List scanners = + List selectedScanners = new ArrayList(allScanners.size()); + // To allow the caller to close the unused ones + List ignoredScanners = + new ArrayList(); // We can only exclude store files based on TTL if minVersions is set to 0. // Otherwise, we might have to return KVs that have technically expired. @@ -406,12 +413,32 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner } if (kvs.shouldUseScanner(scan, store, expiredTimestampCutoff)) { - scanners.add(kvs); + selectedScanners.add(kvs); + } else { + ignoredScanners.add(kvs); } } - return scanners; + return Pair.newPair(selectedScanners, ignoredScanners); + } + + /** + * Calls the function {@link #selectScannersFrom(List)} and + * closes the unselected scanners + * @param scanners from which valid scanners are selected + * @return final list of selected scanners + */ + private List selectScannersFromAndCloseUnSelected( + List scanners) { + Pair, List> selectedResult + = selectScannersFrom(scanners); + for (KeyValueScanner kvs : selectedResult.getSecond()) { + kvs.close(); + } + return selectedResult.getFirst(); } + + @Override public Cell peek() { if (this.heap == null) { @@ -722,8 +749,9 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner List scanners = null; try { flushLock.lock(); - scanners = selectScannersFrom(store.getScanners(flushedStoreFiles, cacheBlocks, get, usePread, - isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt, true)); + scanners = selectScannersFromAndCloseUnSelected(store.getScanners(flushedStoreFiles, + cacheBlocks, get, usePread, isCompaction, matcher, scan.getStartRow(), + scan.getStopRow(), this.readPt, true)); // Clear the current set of flushed store files so that they don't get added again flushedStoreFiles.clear(); } finally { -- 2.2.1