diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java index 1d718bf..7ada9fd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java @@ -88,23 +88,6 @@ public class Scan extends Query { private static final String RAW_ATTR = "_raw_"; - /** - * EXPERT ONLY. - * An integer (not long) indicating to the scanner logic how many times we attempt to retrieve the - * next KV before we schedule a reseek. - * The right value depends on the size of the average KV. A reseek is more efficient when - * it can skip 5-10 KVs or 512B-1KB, or when the next KV is likely found in another HFile block. - * Setting this only has any effect when columns were added with - * {@link #addColumn(byte[], byte[])} - *
{@code
-   * Scan s = new Scan(...);
-   * s.addColumn(...);
-   * s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
-   * }
- * Default is 0 (always reseek). - */ - public static final String HINT_LOOKAHEAD = "_look_ahead_"; - private byte [] startRow = HConstants.EMPTY_START_ROW; private byte [] stopRow = HConstants.EMPTY_END_ROW; private int maxVersions = 1; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java index 5ce6c74..e7ed6d4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java @@ -294,6 +294,11 @@ public class HalfStoreFileReader extends StoreFile.Reader { public boolean isSeeked() { return this.delegate.isSeeked(); } + + @Override + public byte[] getNextIndexedKey() { + return null; + } }; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index 1c7c02b..816822b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -512,6 +512,10 @@ public class HFileReaderV2 extends AbstractHFileReader { super(r, cacheBlocks, pread, isCompaction); } + @Override + public byte[] getNextIndexedKey() { + return nextIndexedKey; + } /** * An internal API function. Seek to the given key, optionally rewinding to * the first key of the block before doing the seek. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java index c0b2047..60cd80a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java @@ -144,4 +144,5 @@ public interface HFileScanner { * Otherwise returns false. */ boolean isSeeked(); + byte[] getNextIndexedKey(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java index b42ff25..0af200e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java @@ -56,10 +56,6 @@ public class ExplicitColumnTracker implements ColumnTracker { private final int maxVersions; private final int minVersions; - // hint for the tracker about how many KVs we will attempt to search via next() - // before we schedule a (re)seek operation - private final int lookAhead; - /** * Contains the list of columns that the ExplicitColumnTracker is tracking. * Each ColumnCount instance also tracks how many versions of the requested @@ -72,7 +68,6 @@ public class ExplicitColumnTracker implements ColumnTracker { * Used to eliminate duplicates. */ private long latestTSOfCurrentColumn; private long oldestStamp; - private int skipCount; /** * Default constructor. @@ -85,10 +80,9 @@ public class ExplicitColumnTracker implements ColumnTracker { * (re)seeking */ public ExplicitColumnTracker(NavigableSet columns, int minVersions, - int maxVersions, long oldestUnexpiredTS, int lookAhead) { + int maxVersions, long oldestUnexpiredTS) { this.maxVersions = maxVersions; this.minVersions = minVersions; - this.lookAhead = lookAhead; this.oldestStamp = oldestUnexpiredTS; this.columns = new ColumnCount[columns.size()]; int i=0; @@ -144,8 +138,7 @@ public class ExplicitColumnTracker implements ColumnTracker { if (ret > 0) { // The current KV is smaller than the column the ExplicitColumnTracker // is interested in, so seek to that column of interest. - return this.skipCount++ < this.lookAhead ? ScanQueryMatcher.MatchCode.SKIP - : ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; + return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; } // The current KV is bigger than the column the ExplicitColumnTracker @@ -154,7 +147,6 @@ public class ExplicitColumnTracker implements ColumnTracker { // column of interest, and check again. if (ret <= -1) { ++this.index; - this.skipCount = 0; if (done()) { // No more to match, do not include, done with this row. return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row @@ -179,7 +171,6 @@ public class ExplicitColumnTracker implements ColumnTracker { if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) { // Done with versions for this column ++this.index; - this.skipCount = 0; resetTS(); if (done()) { // We have served all the requested columns. @@ -198,7 +189,6 @@ public class ExplicitColumnTracker implements ColumnTracker { // Called between every row. public void reset() { this.index = 0; - this.skipCount = 0; this.column = this.columns[this.index]; for(ColumnCount col : this.columns) { col.setCount(0); @@ -238,7 +228,6 @@ public class ExplicitColumnTracker implements ColumnTracker { resetTS(); if (compare <= 0) { ++this.index; - this.skipCount = 0; if (done()) { // Will not hit any more columns in this storefile this.column = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java index f8cc5b4..6bffe50 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java @@ -396,4 +396,8 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner KeyValueScanner getCurrentForTesting() { return current; } + @Override + public byte[] getNextIndexedKey() { + return current == null ? null : current.getNextIndexedKey(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java index babb216..9dc9c8e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java @@ -156,4 +156,5 @@ public interface KeyValueScanner { * @throws IOException */ public boolean seekToLastRow() throws IOException; + byte[] getNextIndexedKey(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java index a892637..5e7cecb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java @@ -67,4 +67,8 @@ public abstract class NonLazyKeyValueScanner implements KeyValueScanner { // Not a file by default. return false; } + @Override + public byte[] getNextIndexedKey() { + return null; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java index 2b56edb..19d3801 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java @@ -203,9 +203,8 @@ public class ScanQueryMatcher { // We can share the ExplicitColumnTracker, diff is we reset // between rows, not between storefiles. - byte[] attr = scan.getAttribute(Scan.HINT_LOOKAHEAD); this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions, - oldestUnexpiredTS, attr == null ? 0 : Bytes.toInt(attr)); + oldestUnexpiredTS); } this.isReversed = scan.isReversed(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java index e528770..ac3d12f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java @@ -475,4 +475,9 @@ public class StoreFileScanner implements KeyValueScanner { } return true; } + + @Override + public byte[] getNextIndexedKey() { + return hfs.getNextIndexedKey(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 853e1bf..b86839c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.client.IsolationLevel; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -478,6 +479,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner prevKV = kv; ScanQueryMatcher.MatchCode qcode = matcher.match(kv); + qcode = optimize(qcode, kv); switch(qcode) { case INCLUDE: case INCLUDE_AND_SEEK_NEXT_ROW: @@ -575,6 +577,44 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner } } + private ScanQueryMatcher.MatchCode optimize(ScanQueryMatcher.MatchCode qcode, KeyValue kv) { + byte[] nextIndexedKey = getNextIndexedKey(); + if (nextIndexedKey == null || nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || store == null) { + return qcode; + } + switch(qcode) { + case INCLUDE_AND_SEEK_NEXT_COL: + case SEEK_NEXT_COL: + { + KeyValue key = matcher.getKeyForNextColumn(kv); + byte[] keyBuf = key.getBuffer(); + int keyOff = key.getKeyOffset(); + int keyLen = key.getKeyLength(); + if (store.getComparator().compareFlatKey(keyBuf, keyOff, keyLen, + nextIndexedKey, 0, nextIndexedKey.length) < 0) { + return qcode == MatchCode.SEEK_NEXT_COL ? MatchCode.SKIP : MatchCode.INCLUDE; + } + break; + } + case INCLUDE_AND_SEEK_NEXT_ROW: + case SEEK_NEXT_ROW: + { + KeyValue key = matcher.getKeyForNextRow(kv); + byte[] keyBuf = key.getBuffer(); + int keyOff = key.getKeyOffset(); + int keyLen = key.getKeyLength(); + if (store.getComparator().compareFlatKey(keyBuf, keyOff, keyLen, + nextIndexedKey, 0, nextIndexedKey.length) < 0) { + return qcode == MatchCode.SEEK_NEXT_ROW ? MatchCode.SKIP : MatchCode.INCLUDE; + } + break; + } + default: + break; + } + return qcode; + } + @Override public boolean next(List outResult) throws IOException { return next(outResult, -1); @@ -778,5 +818,10 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner public long getEstimatedNumberOfKvsScanned() { return this.kvsScanned; } + + @Override + public byte[] getNextIndexedKey() { + return this.heap.getNextIndexedKey(); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java index 401583d..e7a9694 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java @@ -47,9 +47,9 @@ public class TestExplicitColumnTracker { private void runTest(int maxVersions, TreeSet trackColumns, List scannerColumns, - List expected, int lookAhead) throws IOException { + List expected) throws IOException { ColumnTracker exp = new ExplicitColumnTracker( - trackColumns, 0, maxVersions, Long.MIN_VALUE, lookAhead); + trackColumns, 0, maxVersions, Long.MIN_VALUE); //Initialize result @@ -91,7 +91,7 @@ public class TestExplicitColumnTracker { scanner.add(col4); scanner.add(col5); - runTest(maxVersions, columns, scanner, expected, 0); + runTest(maxVersions, columns, scanner, expected); } @Test @@ -143,59 +143,7 @@ public class TestExplicitColumnTracker { scanner.add(col5); //Initialize result - runTest(maxVersions, columns, scanner, expected, 0); - } - - @Test - public void testGet_MultiVersionWithLookAhead() throws IOException{ - //Create tracker - TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); - //Looking for every other - columns.add(col2); - columns.add(col4); - - List expected = new ArrayList(); - expected.add(ScanQueryMatcher.MatchCode.SKIP); - expected.add(ScanQueryMatcher.MatchCode.SKIP); - expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); - - expected.add(ScanQueryMatcher.MatchCode.INCLUDE); // col2; 1st version - expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2; 2nd version - expected.add(ScanQueryMatcher.MatchCode.SKIP); - - expected.add(ScanQueryMatcher.MatchCode.SKIP); - expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); - expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); - - expected.add(ScanQueryMatcher.MatchCode.INCLUDE); // col4; 1st version - expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4; 2nd version - expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); - - expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); - expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); - expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); - int maxVersions = 2; - - //Create "Scanner" - List scanner = new ArrayList(); - scanner.add(col1); - scanner.add(col1); - scanner.add(col1); - scanner.add(col2); - scanner.add(col2); - scanner.add(col2); - scanner.add(col3); - scanner.add(col3); - scanner.add(col3); - scanner.add(col4); - scanner.add(col4); - scanner.add(col4); - scanner.add(col5); - scanner.add(col5); - scanner.add(col5); - - //Initialize result - runTest(maxVersions, columns, scanner, expected, 2); + runTest(maxVersions, columns, scanner, expected); } /** @@ -210,7 +158,7 @@ public class TestExplicitColumnTracker { } ColumnTracker explicit = new ExplicitColumnTracker(columns, 0, maxVersions, - Long.MIN_VALUE, 0); + Long.MIN_VALUE); for (int i = 0; i < 100000; i+=2) { byte [] col = Bytes.toBytes("col"+i); ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(), @@ -239,7 +187,7 @@ public class TestExplicitColumnTracker { new ScanQueryMatcher.MatchCode[] { ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.SEEK_NEXT_COL }); - runTest(1, columns, scanner, expected, 0); + runTest(1, columns, scanner, expected); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java index 4f07e2d..655e6fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java @@ -142,26 +142,6 @@ public class TestQueryMatcher extends HBaseTestCase { _testMatch_ExplicitColumns(scan, expected); } - public void testMatch_ExplicitColumnsWithLookAhead() - throws IOException { - //Moving up from the Tracker by using Gets and List instead - //of just byte [] - - //Expected result - List expected = new ArrayList(); - expected.add(ScanQueryMatcher.MatchCode.SKIP); - expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); - expected.add(ScanQueryMatcher.MatchCode.SKIP); - expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); - expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); - expected.add(ScanQueryMatcher.MatchCode.DONE); - - Scan s = new Scan(scan); - s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2)); - _testMatch_ExplicitColumns(s, expected); - } - - public void testMatch_Wildcard() throws IOException { //Moving up from the Tracker by using Gets and List instead diff --git a/pom.xml b/pom.xml index fa388a3..27209ee 100644 --- a/pom.xml +++ b/pom.xml @@ -985,7 +985,7 @@ 3.0.3 ${compileSource} - 2.2.0 + 2.4.1 1.2.1 1.2 1.7