Index: src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java (revision 0) @@ -0,0 +1,170 @@ +/** + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * under the License. + */ +package org.apache.hadoop.hbase.filter; + +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueTestUtil; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBuffers; +import org.junit.*; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + */ +@Category(MediumTests.class) +public class TestFuzzyRowAndColumnRangeFilter { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final Log LOG = LogFactory.getLog(this.getClass()); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + @Test + public void Test() throws Exception { + String cf = "f"; + String table = "TestFuzzyAndColumnRangeFilterClient"; + HTable ht = TEST_UTIL.createTable(Bytes.toBytes(table), + Bytes.toBytes(cf), Integer.MAX_VALUE); + + // 10 byte row key - (2 bytes 4 bytes 4 bytes) + // 4 byte qualifier + // 4 byte value + + for (int i1 = 0; i1 < 2; i1++) { + for (int i2 = 0; i2 < 5; i2++) { + byte[] rk = new byte[10]; + + ChannelBuffer buf = ChannelBuffers.wrappedBuffer(rk); + buf.clear(); + buf.writeShort((short) 2); + buf.writeInt(i1); + buf.writeInt(i2); + + for (int c = 0; c < 5; c++) { + byte[] cq = new byte[4]; + Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4); + + Put p = new Put(rk); + p.setDurability(Durability.SKIP_WAL); + p.add(cf.getBytes(), cq, Bytes.toBytes(c)); + ht.put(p); + LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + + Bytes.toStringBinary(cq)); + } + } + } + + TEST_UTIL.flush(); + + // test passes + runTest(ht, 0, 10); + + // test fails + runTest(ht, 1, 8); + } + + private void runTest(HTable hTable, int cqStart, int expectedSize) throws IOException { + // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1] + byte[] fuzzyKey = new byte[10]; + ChannelBuffer buf = ChannelBuffers.wrappedBuffer(fuzzyKey); + buf.clear(); + buf.writeShort((short) 2); + for (int i = 0; i < 4; i++) + buf.writeByte((short)63); + buf.writeInt((short)1); + + byte[] mask = new byte[] {0 , 0, 1, 1, 1, 1, 0, 0, 0, 0}; + + Pair pair = new Pair(fuzzyKey, mask); + FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(Lists.newArrayList(pair)); + ColumnRangeFilter columnRangeFilter = new ColumnRangeFilter(Bytes.toBytes(cqStart), true + , Bytes.toBytes(4), true); + //regular test + runScanner(hTable, expectedSize, fuzzyRowFilter, columnRangeFilter); + //reverse filter order test + runScanner(hTable, expectedSize, columnRangeFilter, fuzzyRowFilter); + } + + private void runScanner(HTable hTable, int expectedSize, Filter... filters) throws IOException { + String cf = "f"; + Scan scan = new Scan(); + scan.addFamily(cf.getBytes()); + FilterList filterList = new FilterList(filters); + scan.setFilter(filterList); + + ResultScanner scanner = hTable.getScanner(scan); + List results = new ArrayList(); + Result result; + long timeBeforeScan = System.currentTimeMillis(); + while ((result = scanner.next()) != null) { + for (KeyValue kv : result.list()) { + LOG.info("Got rk: " + Bytes.toStringBinary(kv.getRow()) + " cq: " + + Bytes.toStringBinary(kv.getQualifier())); + results.add(kv); + } + } + long scanTime = System.currentTimeMillis() - timeBeforeScan; + scanner.close(); + + LOG.info("scan time = " + scanTime + "ms"); + LOG.info("found " + results.size() + " results"); + + assertEquals(expectedSize, results.size()); + } +} \ No newline at end of file Index: src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java (revision 1511065) +++ src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java (working copy) @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.util.Bytes; import org.junit.experimental.categories.Category; @@ -320,6 +321,11 @@ Filter filterMinHint = new FilterBase() { @Override + public ReturnCode filterKeyValue(KeyValue ignored) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + + @Override public KeyValue getNextKeyHint(KeyValue currentKV) { return minKeyValue; } @@ -333,6 +339,11 @@ Filter filterMaxHint = new FilterBase() { @Override + public ReturnCode filterKeyValue(KeyValue ignored) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + + @Override public KeyValue getNextKeyHint(KeyValue currentKV) { return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null); } @@ -369,32 +380,37 @@ // MUST PASS ALL - // Should take the max if given two hints + // Should take the first hint filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + minKeyValue)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList(new Filter [] { filterMaxHint, filterMinHint } )); + filterList.filterKeyValue(null); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); - // Should have max hint even if a filter has no hint + // Should have first hint even if a filter has no hint filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList( - new Filter [] { filterMinHint, filterMaxHint, filterNoHint } )); + new Filter [] { filterNoHint, filterMinHint, filterMaxHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - maxKeyValue)); + minKeyValue)); filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); + filterList.filterKeyValue(null); filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(new Filter [] { filterNoHint, filterMinHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), minKeyValue)); - - // Should give min hint if its the only one - filterList = new FilterList(Operator.MUST_PASS_ALL, - Arrays.asList(new Filter [] { filterNoHint, filterMinHint } )); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - minKeyValue)); } /** Index: src/main/java/org/apache/hadoop/hbase/filter/FilterList.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/filter/FilterList.java (revision 1511065) +++ src/main/java/org/apache/hadoop/hbase/filter/FilterList.java (working copy) @@ -63,6 +63,7 @@ private static final int MAX_LOG_FILTERS = 5; private Operator operator = Operator.MUST_PASS_ALL; private List filters = new ArrayList(); + private Filter seekHintFilter = null; /** Reference KeyValue used by {@link #transform(KeyValue)} for validation purpose. */ private KeyValue referenceKV = null; @@ -166,6 +167,7 @@ for (Filter filter : filters) { filter.reset(); } + seekHintFilter = null; } @Override @@ -235,6 +237,9 @@ case INCLUDE: transformed = filter.transform(transformed); continue; + case SEEK_NEXT_USING_HINT: + seekHintFilter = filter; + return code; default: return code; } @@ -243,7 +248,8 @@ continue; } - switch (filter.filterKeyValue(v)) { + ReturnCode code = filter.filterKeyValue(v); + switch (code) { case INCLUDE: if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) { rc = ReturnCode.INCLUDE; @@ -332,9 +338,14 @@ @Override public KeyValue getNextKeyHint(KeyValue currentKV) { KeyValue keyHint = null; + if (operator == Operator.MUST_PASS_ALL) { + keyHint = seekHintFilter.getNextKeyHint(currentKV); + return keyHint; + } + for (Filter filter : filters) { KeyValue curKeyHint = filter.getNextKeyHint(currentKV); - if (curKeyHint == null && operator == Operator.MUST_PASS_ONE) { + if (curKeyHint == null) { // If we ever don't have a hint and this is must-pass-one, then no hint return null; } @@ -344,13 +355,7 @@ keyHint = curKeyHint; continue; } - // There is an existing hint - if (operator == Operator.MUST_PASS_ALL && - KeyValue.COMPARATOR.compare(keyHint, curKeyHint) < 0) { - // If all conditions must pass, we can keep the max hint - keyHint = curKeyHint; - } else if (operator == Operator.MUST_PASS_ONE && - KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) { + if (KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) { // If any condition can pass, we need to keep the min hint keyHint = curKeyHint; }