diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 1d718bf..7ada9fd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -88,23 +88,6 @@ public class Scan extends Query {
private static final String RAW_ATTR = "_raw_";
- /**
- * EXPERT ONLY.
- * An integer (not long) indicating to the scanner logic how many times we attempt to retrieve the
- * next KV before we schedule a reseek.
- * The right value depends on the size of the average KV. A reseek is more efficient when
- * it can skip 5-10 KVs or 512B-1KB, or when the next KV is likely found in another HFile block.
- * Setting this only has any effect when columns were added with
- * {@link #addColumn(byte[], byte[])}
- *
{@code
- * Scan s = new Scan(...);
- * s.addColumn(...);
- * s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
- * }
- * Default is 0 (always reseek).
- */
- public static final String HINT_LOOKAHEAD = "_look_ahead_";
-
private byte [] startRow = HConstants.EMPTY_START_ROW;
private byte [] stopRow = HConstants.EMPTY_END_ROW;
private int maxVersions = 1;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index 5ce6c74..e7ed6d4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -294,6 +294,11 @@ public class HalfStoreFileReader extends StoreFile.Reader {
public boolean isSeeked() {
return this.delegate.isSeeked();
}
+
+ @Override
+ public byte[] getNextIndexedKey() {
+ return null;
+ }
};
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index 1c7c02b..816822b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -512,6 +512,10 @@ public class HFileReaderV2 extends AbstractHFileReader {
super(r, cacheBlocks, pread, isCompaction);
}
+ @Override
+ public byte[] getNextIndexedKey() {
+ return nextIndexedKey;
+ }
/**
* An internal API function. Seek to the given key, optionally rewinding to
* the first key of the block before doing the seek.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
index c0b2047..60cd80a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
@@ -144,4 +144,5 @@ public interface HFileScanner {
* Otherwise returns false.
*/
boolean isSeeked();
+ byte[] getNextIndexedKey();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index b42ff25..0af200e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -56,10 +56,6 @@ public class ExplicitColumnTracker implements ColumnTracker {
private final int maxVersions;
private final int minVersions;
- // hint for the tracker about how many KVs we will attempt to search via next()
- // before we schedule a (re)seek operation
- private final int lookAhead;
-
/**
* Contains the list of columns that the ExplicitColumnTracker is tracking.
* Each ColumnCount instance also tracks how many versions of the requested
@@ -72,7 +68,6 @@ public class ExplicitColumnTracker implements ColumnTracker {
* Used to eliminate duplicates. */
private long latestTSOfCurrentColumn;
private long oldestStamp;
- private int skipCount;
/**
* Default constructor.
@@ -85,10 +80,9 @@ public class ExplicitColumnTracker implements ColumnTracker {
* (re)seeking
*/
public ExplicitColumnTracker(NavigableSet columns, int minVersions,
- int maxVersions, long oldestUnexpiredTS, int lookAhead) {
+ int maxVersions, long oldestUnexpiredTS) {
this.maxVersions = maxVersions;
this.minVersions = minVersions;
- this.lookAhead = lookAhead;
this.oldestStamp = oldestUnexpiredTS;
this.columns = new ColumnCount[columns.size()];
int i=0;
@@ -144,8 +138,7 @@ public class ExplicitColumnTracker implements ColumnTracker {
if (ret > 0) {
// The current KV is smaller than the column the ExplicitColumnTracker
// is interested in, so seek to that column of interest.
- return this.skipCount++ < this.lookAhead ? ScanQueryMatcher.MatchCode.SKIP
- : ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
+ return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
}
// The current KV is bigger than the column the ExplicitColumnTracker
@@ -154,7 +147,6 @@ public class ExplicitColumnTracker implements ColumnTracker {
// column of interest, and check again.
if (ret <= -1) {
++this.index;
- this.skipCount = 0;
if (done()) {
// No more to match, do not include, done with this row.
return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
@@ -179,7 +171,6 @@ public class ExplicitColumnTracker implements ColumnTracker {
if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
// Done with versions for this column
++this.index;
- this.skipCount = 0;
resetTS();
if (done()) {
// We have served all the requested columns.
@@ -198,7 +189,6 @@ public class ExplicitColumnTracker implements ColumnTracker {
// Called between every row.
public void reset() {
this.index = 0;
- this.skipCount = 0;
this.column = this.columns[this.index];
for(ColumnCount col : this.columns) {
col.setCount(0);
@@ -238,7 +228,6 @@ public class ExplicitColumnTracker implements ColumnTracker {
resetTS();
if (compare <= 0) {
++this.index;
- this.skipCount = 0;
if (done()) {
// Will not hit any more columns in this storefile
this.column = null;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index f8cc5b4..6bffe50 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -396,4 +396,8 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner
KeyValueScanner getCurrentForTesting() {
return current;
}
+ @Override
+ public byte[] getNextIndexedKey() {
+ return current == null ? null : current.getNextIndexedKey();
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
index babb216..9dc9c8e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
@@ -156,4 +156,5 @@ public interface KeyValueScanner {
* @throws IOException
*/
public boolean seekToLastRow() throws IOException;
+ byte[] getNextIndexedKey();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
index a892637..5e7cecb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
@@ -67,4 +67,8 @@ public abstract class NonLazyKeyValueScanner implements KeyValueScanner {
// Not a file by default.
return false;
}
+ @Override
+ public byte[] getNextIndexedKey() {
+ return null;
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 2b56edb..19d3801 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -203,9 +203,8 @@ public class ScanQueryMatcher {
// We can share the ExplicitColumnTracker, diff is we reset
// between rows, not between storefiles.
- byte[] attr = scan.getAttribute(Scan.HINT_LOOKAHEAD);
this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions,
- oldestUnexpiredTS, attr == null ? 0 : Bytes.toInt(attr));
+ oldestUnexpiredTS);
}
this.isReversed = scan.isReversed();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index e528770..ac3d12f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -475,4 +475,9 @@ public class StoreFileScanner implements KeyValueScanner {
}
return true;
}
+
+ @Override
+ public byte[] getNextIndexedKey() {
+ return hfs.getNextIndexedKey();
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 853e1bf..b86839c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.client.IsolationLevel;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -478,6 +479,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
prevKV = kv;
ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
+ qcode = optimize(qcode, kv);
switch(qcode) {
case INCLUDE:
case INCLUDE_AND_SEEK_NEXT_ROW:
@@ -575,6 +577,44 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
}
}
+ private ScanQueryMatcher.MatchCode optimize(ScanQueryMatcher.MatchCode qcode, KeyValue kv) {
+ byte[] nextIndexedKey = getNextIndexedKey();
+ if (nextIndexedKey == null || nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || store == null) {
+ return qcode;
+ }
+ switch(qcode) {
+ case INCLUDE_AND_SEEK_NEXT_COL:
+ case SEEK_NEXT_COL:
+ {
+ KeyValue key = matcher.getKeyForNextColumn(kv);
+ byte[] keyBuf = key.getBuffer();
+ int keyOff = key.getKeyOffset();
+ int keyLen = key.getKeyLength();
+ if (store.getComparator().compareFlatKey(keyBuf, keyOff, keyLen,
+ nextIndexedKey, 0, nextIndexedKey.length) < 0) {
+ return qcode == MatchCode.SEEK_NEXT_COL ? MatchCode.SKIP : MatchCode.INCLUDE;
+ }
+ break;
+ }
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ case SEEK_NEXT_ROW:
+ {
+ KeyValue key = matcher.getKeyForNextRow(kv);
+ byte[] keyBuf = key.getBuffer();
+ int keyOff = key.getKeyOffset();
+ int keyLen = key.getKeyLength();
+ if (store.getComparator().compareFlatKey(keyBuf, keyOff, keyLen,
+ nextIndexedKey, 0, nextIndexedKey.length) < 0) {
+ return qcode == MatchCode.SEEK_NEXT_ROW ? MatchCode.SKIP : MatchCode.INCLUDE;
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ return qcode;
+ }
+
@Override
public boolean next(List outResult) throws IOException {
return next(outResult, -1);
@@ -778,5 +818,10 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
public long getEstimatedNumberOfKvsScanned() {
return this.kvsScanned;
}
+
+ @Override
+ public byte[] getNextIndexedKey() {
+ return this.heap.getNextIndexedKey();
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
index 401583d..e7a9694 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
@@ -47,9 +47,9 @@ public class TestExplicitColumnTracker {
private void runTest(int maxVersions,
TreeSet trackColumns,
List scannerColumns,
- List expected, int lookAhead) throws IOException {
+ List expected) throws IOException {
ColumnTracker exp = new ExplicitColumnTracker(
- trackColumns, 0, maxVersions, Long.MIN_VALUE, lookAhead);
+ trackColumns, 0, maxVersions, Long.MIN_VALUE);
//Initialize result
@@ -91,7 +91,7 @@ public class TestExplicitColumnTracker {
scanner.add(col4);
scanner.add(col5);
- runTest(maxVersions, columns, scanner, expected, 0);
+ runTest(maxVersions, columns, scanner, expected);
}
@Test
@@ -143,59 +143,7 @@ public class TestExplicitColumnTracker {
scanner.add(col5);
//Initialize result
- runTest(maxVersions, columns, scanner, expected, 0);
- }
-
- @Test
- public void testGet_MultiVersionWithLookAhead() throws IOException{
- //Create tracker
- TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR);
- //Looking for every other
- columns.add(col2);
- columns.add(col4);
-
- List expected = new ArrayList();
- expected.add(ScanQueryMatcher.MatchCode.SKIP);
- expected.add(ScanQueryMatcher.MatchCode.SKIP);
- expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
-
- expected.add(ScanQueryMatcher.MatchCode.INCLUDE); // col2; 1st version
- expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2; 2nd version
- expected.add(ScanQueryMatcher.MatchCode.SKIP);
-
- expected.add(ScanQueryMatcher.MatchCode.SKIP);
- expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
- expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
-
- expected.add(ScanQueryMatcher.MatchCode.INCLUDE); // col4; 1st version
- expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4; 2nd version
- expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
-
- expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
- expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
- expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
- int maxVersions = 2;
-
- //Create "Scanner"
- List scanner = new ArrayList();
- scanner.add(col1);
- scanner.add(col1);
- scanner.add(col1);
- scanner.add(col2);
- scanner.add(col2);
- scanner.add(col2);
- scanner.add(col3);
- scanner.add(col3);
- scanner.add(col3);
- scanner.add(col4);
- scanner.add(col4);
- scanner.add(col4);
- scanner.add(col5);
- scanner.add(col5);
- scanner.add(col5);
-
- //Initialize result
- runTest(maxVersions, columns, scanner, expected, 2);
+ runTest(maxVersions, columns, scanner, expected);
}
/**
@@ -210,7 +158,7 @@ public class TestExplicitColumnTracker {
}
ColumnTracker explicit = new ExplicitColumnTracker(columns, 0, maxVersions,
- Long.MIN_VALUE, 0);
+ Long.MIN_VALUE);
for (int i = 0; i < 100000; i+=2) {
byte [] col = Bytes.toBytes("col"+i);
ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
@@ -239,7 +187,7 @@ public class TestExplicitColumnTracker {
new ScanQueryMatcher.MatchCode[] {
ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
ScanQueryMatcher.MatchCode.SEEK_NEXT_COL });
- runTest(1, columns, scanner, expected, 0);
+ runTest(1, columns, scanner, expected);
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
index 4f07e2d..655e6fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
@@ -142,26 +142,6 @@ public class TestQueryMatcher extends HBaseTestCase {
_testMatch_ExplicitColumns(scan, expected);
}
- public void testMatch_ExplicitColumnsWithLookAhead()
- throws IOException {
- //Moving up from the Tracker by using Gets and List instead
- //of just byte []
-
- //Expected result
- List expected = new ArrayList();
- expected.add(ScanQueryMatcher.MatchCode.SKIP);
- expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
- expected.add(ScanQueryMatcher.MatchCode.SKIP);
- expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
- expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
- expected.add(ScanQueryMatcher.MatchCode.DONE);
-
- Scan s = new Scan(scan);
- s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
- _testMatch_ExplicitColumns(s, expected);
- }
-
-
public void testMatch_Wildcard()
throws IOException {
//Moving up from the Tracker by using Gets and List instead
diff --git a/pom.xml b/pom.xml
index fa388a3..27209ee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -985,7 +985,7 @@
3.0.3
${compileSource}
- 2.2.0
+ 2.4.1
1.2.1
1.2
1.7
|