### Eclipse Workspace Patch 1.0 #P apache-trunk Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java (revision 1552207) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java (working copy) @@ -187,7 +187,7 @@ i = nextReadableNum.getFirst(); int qualNum = nextReadableNum.getSecond(); if (qualNum + 1 < QUALSIZE) { - kvHeap.backwardSeek(makeKV(i, qualNum + 1)); + kvHeap.backwardSeek(makeKV(i, qualNum + 1), false); nextReadableNum = getNextReadableNumWithBackwardScan(i, qualNum + 1, readPoint); if (nextReadableNum == null) break; @@ -463,7 +463,7 @@ for (int j = 0; j < QUALSIZE; j++) { if (j % 2 == 1 && (j + 1) < QUALSIZE) { j = j + 1; - kvHeap.backwardSeek(makeKV(i, j)); + kvHeap.backwardSeek(makeKV(i, j), false); } assertEquals(makeKV(i, j), kvHeap.peek()); kvHeap.next(); @@ -502,7 +502,7 @@ } else { KeyValue startKey = KeyValue.createFirstOnRow(startRow); for (KeyValueScanner scanner : scanners) { - scanner.backwardSeek(startKey); + scanner.backwardSeek(startKey, false); } } } @@ -521,17 +521,18 @@ // Test backward seek in three cases // Case1: seek in the same row in backwardSeek KeyValue seekKey = makeKV(ROWSIZE - 2, QUALSIZE - 2); - assertTrue(scanner.backwardSeek(seekKey)); + assertTrue(scanner.backwardSeek(seekKey, false)); assertEquals(seekKey, scanner.peek()); // Case2: seek to the previous row in backwardSeek int seekRowNum = ROWSIZE - 2; - assertTrue(scanner.backwardSeek(KeyValue.createLastOnRow(ROWS[seekRowNum]))); + assertTrue(scanner.backwardSeek(KeyValue.createLastOnRow(ROWS[seekRowNum]), + false)); KeyValue expectedKey = makeKV(seekRowNum - 1, 0); assertEquals(expectedKey, scanner.peek()); // Case3: unable to backward seek - assertFalse(scanner.backwardSeek(KeyValue.createLastOnRow(ROWS[0]))); + assertFalse(scanner.backwardSeek(KeyValue.createLastOnRow(ROWS[0]), false)); assertEquals(null, scanner.peek()); // Test seek to previous row @@ -562,7 +563,7 @@ // Case1: seek in the same row in backwardSeek expectedKey = getNextReadableKeyValueWithBackwardScan(ROWSIZE - 2, QUALSIZE - 2, readPoint); - assertEquals(expectedKey != null, scanner.backwardSeek(expectedKey)); + assertEquals(expectedKey != null, scanner.backwardSeek(expectedKey, false)); assertEquals(expectedKey, scanner.peek()); // Case2: seek to the previous row in backwardSeek @@ -570,7 +571,7 @@ KeyValue seekKey = KeyValue.createLastOnRow(ROWS[seekRowNum]); expectedKey = getNextReadableKeyValueWithBackwardScan(seekRowNum - 1, 0, readPoint); - assertEquals(expectedKey != null, scanner.backwardSeek(seekKey)); + assertEquals(expectedKey != null, scanner.backwardSeek(seekKey, false)); assertEquals(expectedKey, scanner.peek()); // Test seek to previous row Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedStoreScanner.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedStoreScanner.java (revision 1552207) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedStoreScanner.java (working copy) @@ -46,8 +46,7 @@ * @throws IOException */ ReversedStoreScanner(Store store, ScanInfo scanInfo, Scan scan, - NavigableSet columns, long readPt) - throws IOException { + NavigableSet columns, long readPt) throws IOException { super(store, scanInfo, scan, columns, readPt); } @@ -78,7 +77,7 @@ } } else { for (KeyValueScanner scanner : scanners) { - scanner.backwardSeek(seekKey); + scanner.backwardSeek(seekKey, isLazy); } } } @@ -93,7 +92,8 @@ */ @Override protected boolean seekAsDirection(KeyValue kv) throws IOException { - return backwardSeek(kv); + boolean isLazy = explicitColumnQuery && lazySeekEnabledGlobally; + return backwardSeek(kv, isLazy); } @Override @@ -104,8 +104,8 @@ assert prevKV == null || comparator == null || comparator.compareRows(kv, prevKV) < 0 || (comparator.matchingRows(kv, prevKV) && comparator.compare(kv, prevKV) >= 0) : "Key " + prevKV - + " followed by a " + "error order key " + kv + " in cf " + store - + " in reversed scan"; + + " followed by a " + "error order key " + kv + " in cf " + store + + " in reversed scan"; } @Override @@ -131,13 +131,13 @@ } } - + @Override - public boolean backwardSeek(KeyValue key) throws IOException { + public boolean backwardSeek(KeyValue key, boolean isLazy) throws IOException { lock.lock(); try { checkReseek(); - return this.heap.backwardSeek(key); + return this.heap.backwardSeek(key, isLazy); } finally { lock.unlock(); } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (revision 1552207) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (working copy) @@ -923,7 +923,7 @@ * the scanner to the previous row of given key */ @Override - public synchronized boolean backwardSeek(KeyValue key) { + public synchronized boolean backwardSeek(KeyValue key, boolean isLazy) { seek(key); if (peek() == null || comparator.compareRows(peek(), key) > 0) { return seekToPreviousRow(key); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedKeyValueHeap.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedKeyValueHeap.java (revision 1552207) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedKeyValueHeap.java (working copy) @@ -97,7 +97,8 @@ } @Override - public boolean backwardSeek(KeyValue seekKey) throws IOException { + public boolean backwardSeek(KeyValue seekKey, boolean isLazy) + throws IOException { if (current == null) { return false; } @@ -114,7 +115,7 @@ current = pollRealKV(); return current != null; } - if (!scanner.backwardSeek(seekKey)) { + if (!scanner.backwardSeek(seekKey, isLazy)) { scanner.close(); } else { heap.add(scanner); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java (revision 1552207) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java (working copy) @@ -356,7 +356,7 @@ // Compare the current scanner to the next scanner. We try to avoid // putting the current one back into the heap if possible. KeyValue nextKV = nextEarliestScanner.peek(); - if (nextKV == null || comparator.compare(curKV, nextKV) < 0) { + if (nextKV == null || comparator.compare(kvScanner, nextEarliestScanner) < 0) { // We already have the scanner with the earliest KV, so return it. return kvScanner; } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java (revision 1552207) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java (working copy) @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -170,12 +171,33 @@ for (int maxVersions : MAX_VERSIONS_VALUES) { for (boolean lazySeekEnabled : new boolean[] { false, true }) { testScan(columnArr, lazySeekEnabled, rowRange[0], rowRange[1], - maxVersions); + maxVersions, false); } } } } + assertSeekSavings(); + totalSeekLazy = 0; + totalSeekDiligent = 0; + for (int[] columnArr : COLUMN_SETS) { + for (int[] rowRange : ROW_RANGES) { + for (int maxVersions : MAX_VERSIONS_VALUES) { + for (boolean lazySeekEnabled : new boolean[] { false, true }) { + if (rowRange[0] == rowRange[1] && rowRange[0] != -1) { + // It's a get scan, and doesn't support reversed scan. + continue; + } + testScan(columnArr, lazySeekEnabled, rowRange[0], rowRange[1], + maxVersions, true); + } + } + } + } + assertSeekSavings(); + } + + private void assertSeekSavings() { final double seekSavings = 1 - totalSeekLazy * 1.0 / totalSeekDiligent; System.err.println("For bloom=" + bloomType + ", compr=" + comprAlgo + " total seeks without optimization: " + totalSeekDiligent @@ -194,10 +216,11 @@ } private void testScan(final int[] columnArr, final boolean lazySeekEnabled, - final int startRow, final int endRow, int maxVersions) + final int startRow, final int endRow, int maxVersions, boolean isReversed) throws IOException { StoreScanner.enableLazySeekGlobally(lazySeekEnabled); final Scan scan = new Scan(); + scan.setReversed(isReversed); final Set qualSet = new HashSet(); for (int iColumn : columnArr) { String qualStr = getQualStr(iColumn); @@ -205,13 +228,23 @@ qualSet.add(qualStr); } scan.setMaxVersions(maxVersions); - scan.setStartRow(rowBytes(startRow)); + if (!isReversed) { + scan.setStartRow(rowBytes(startRow)); + } else { + final byte[] scannerStopRow = rowBytes(startRow + - (startRow != endRow ? 1 : 0)); + scan.setStopRow(scannerStopRow); + } // Adjust for the fact that for multi-row queries the end row is exclusive. { final byte[] scannerStopRow = rowBytes(endRow + (startRow != endRow ? 1 : 0)); - scan.setStopRow(scannerStopRow); + if (!isReversed) { + scan.setStopRow(scannerStopRow); + } else { + scan.setStartRow(rowBytes(endRow)); + } } final long initialSeekCount = StoreFileScanner.getSeekCount(); @@ -229,8 +262,8 @@ results.clear(); } while (hasNext); - List filteredKVs = filterExpectedResults(qualSet, - rowBytes(startRow), rowBytes(endRow), maxVersions); + List filteredKVs = filterExpectedResults(qualSet, rowBytes(startRow), + rowBytes(endRow), maxVersions, isReversed); final String rowRestrictionStr = (startRow == -1 && endRow == -1) ? "all rows" : ( startRow == endRow ? ("row=" + startRow) : ("startRow=" @@ -239,10 +272,10 @@ columnArr.length == 0 ? "all columns" : ("columns=" + Arrays.toString(columnArr)); final String testDesc = - "Bloom=" + bloomType + ", compr=" + comprAlgo + ", " - + (scan.isGetScan() ? "Get" : "Scan") + ": " - + columnRestrictionStr + ", " + rowRestrictionStr - + ", maxVersions=" + maxVersions + ", lazySeek=" + lazySeekEnabled; + "Bloom=" + bloomType + ", compr=" + comprAlgo + ", " + + (scan.isGetScan() ? "Get" : "Scan") + ": " + + columnRestrictionStr + ", " + rowRestrictionStr + + ", maxVersions=" + maxVersions + ", lazySeek=" + lazySeekEnabled; long seekCount = StoreFileScanner.getSeekCount() - initialSeekCount; if (VERBOSE) { System.err.println("Seek count: " + seekCount + ", KVs returned: " @@ -258,10 +291,27 @@ } private List filterExpectedResults(Set qualSet, - byte[] startRow, byte[] endRow, int maxVersions) { + byte[] startRow, byte[] endRow, int maxVersions, boolean reversedScan) { final List filteredKVs = new ArrayList(); final Map verCount = new HashMap(); - for (Cell kv : expectedKVs) { + List sortedKVs = expectedKVs; + if (reversedScan) { + sortedKVs = new ArrayList(); + sortedKVs.addAll(expectedKVs); + Collections.sort(sortedKVs, new Comparator() { + @Override + public int compare(Cell c1, Cell c2) { + int rowComparison = Bytes.compareTo(c1.getRowArray(), + c1.getRowOffset(), c1.getRowLength(), c2.getRowArray(), + c2.getRowOffset(), c2.getRowLength()); + if (rowComparison != 0) { + return -rowComparison; + } + return KeyValue.COMPARATOR.compare(c1, c2); + } + }); + } + for (Cell kv : sortedKVs) { if (startRow.length > 0 && Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), startRow, 0, startRow.length) < 0) { Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonReversedNonLazyKeyValueScanner.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonReversedNonLazyKeyValueScanner.java (revision 1552207) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonReversedNonLazyKeyValueScanner.java (working copy) @@ -34,7 +34,7 @@ NonLazyKeyValueScanner { @Override - public boolean backwardSeek(KeyValue key) throws IOException { + public boolean backwardSeek(KeyValue key, boolean isLazy) throws IOException { throw new NotImplementedException("backwardSeek must not be called on a " + "non-reversed scanner"); } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (revision 1552207) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (working copy) @@ -51,6 +51,7 @@ private boolean realSeekDone; private boolean delayedReseek; + private boolean delayedBackwardSeek; private KeyValue delayedSeekKV; private boolean enforceMVCC = false; @@ -369,9 +370,11 @@ public void enforceSeek() throws IOException { if (realSeekDone) return; - + if (delayedReseek) { reseek(delayedSeekKV); + } else if (delayedBackwardSeek) { + backwardSeek(delayedSeekKV, false); } else { seek(delayedSeekKV); } @@ -462,7 +465,12 @@ } @Override - public boolean backwardSeek(KeyValue key) throws IOException { + public boolean backwardSeek(KeyValue key, boolean isLazy) throws IOException { + if(isLazy){ + delayedBackwardSeek = true; + return requestSeek(key, false, true); + } + delayedBackwardSeek = false; seek(key); if (cur == null || Bytes.compareTo(cur.getBuffer(), cur.getRowOffset(), Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java (revision 1552207) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java (working copy) @@ -133,11 +133,14 @@ * previous row of specified KeyValue * * @param key seek KeyValue + * @param isLazy if true, only does a seek operation after checking that it is + * really necessary for the row/column combination specified by the + * kv parameter. if false, do the real seek always * @return true if the scanner is at the valid KeyValue, false if such * KeyValue does not exist * */ - public boolean backwardSeek(KeyValue key) throws IOException; + public boolean backwardSeek(KeyValue key, boolean isLazy) throws IOException; /** * Seek the scanner at the first KeyValue of the row which is the previous row