Index: hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java (revision 0) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowAndColumnRangeFilter.java (revision 0) @@ -0,0 +1,170 @@ +/** + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * under the License. + */ +package org.apache.hadoop.hbase.filter; + +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueTestUtil; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBuffers; +import org.junit.*; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + */ +@Category(MediumTests.class) +public class TestFuzzyRowAndColumnRangeFilter { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final Log LOG = LogFactory.getLog(this.getClass()); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + @Test + public void Test() throws Exception { + String cf = "f"; + String table = "TestFuzzyAndColumnRangeFilterClient"; + HTable ht = TEST_UTIL.createTable(Bytes.toBytes(table), + Bytes.toBytes(cf), Integer.MAX_VALUE); + + // 10 byte row key - (2 bytes 4 bytes 4 bytes) + // 4 byte qualifier + // 4 byte value + + for (int i1 = 0; i1 < 2; i1++) { + for (int i2 = 0; i2 < 5; i2++) { + byte[] rk = new byte[10]; + + ChannelBuffer buf = ChannelBuffers.wrappedBuffer(rk); + buf.clear(); + buf.writeShort((short) 2); + buf.writeInt(i1); + buf.writeInt(i2); + + for (int c = 0; c < 5; c++) { + byte[] cq = new byte[4]; + Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4); + + Put p = new Put(rk); + p.setDurability(Durability.SKIP_WAL); + p.add(cf.getBytes(), cq, Bytes.toBytes(c)); + ht.put(p); + LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + + Bytes.toStringBinary(cq)); + } + } + } + + TEST_UTIL.flush(); + + // test passes + runTest(ht, 0, 10); + + // test fails + runTest(ht, 1, 8); + } + + private void runTest(HTable hTable, int cqStart, int expectedSize) throws IOException { + // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1] + byte[] fuzzyKey = new byte[10]; + ChannelBuffer buf = ChannelBuffers.wrappedBuffer(fuzzyKey); + buf.clear(); + buf.writeShort((short) 2); + for (int i = 0; i < 4; i++) + buf.writeByte((short)63); + buf.writeInt((short)1); + + byte[] mask = new byte[] {0 , 0, 1, 1, 1, 1, 0, 0, 0, 0}; + + Pair pair = new Pair(fuzzyKey, mask); + FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(Lists.newArrayList(pair)); + ColumnRangeFilter columnRangeFilter = new ColumnRangeFilter(Bytes.toBytes(cqStart), true + , Bytes.toBytes(4), true); + //regular test + runScanner(hTable, expectedSize, fuzzyRowFilter, columnRangeFilter); + //reverse filter order test + runScanner(hTable, expectedSize, columnRangeFilter, fuzzyRowFilter); + } + + private void runScanner(HTable hTable, int expectedSize, Filter... filters) throws IOException { + String cf = "f"; + Scan scan = new Scan(); + scan.addFamily(cf.getBytes()); + FilterList filterList = new FilterList(filters); + scan.setFilter(filterList); + + ResultScanner scanner = hTable.getScanner(scan); + List results = new ArrayList(); + Result result; + long timeBeforeScan = System.currentTimeMillis(); + while ((result = scanner.next()) != null) { + for (KeyValue kv : result.list()) { + LOG.info("Got rk: " + Bytes.toStringBinary(kv.getRow()) + " cq: " + + Bytes.toStringBinary(kv.getQualifier())); + results.add(kv); + } + } + long scanTime = System.currentTimeMillis() - timeBeforeScan; + scanner.close(); + + LOG.info("scan time = " + scanTime + "ms"); + LOG.info("found " + results.size() + " results"); + + assertEquals(expectedSize, results.size()); + } +} \ No newline at end of file Index: hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java (revision 1507535) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java (working copy) @@ -352,6 +352,11 @@ Filter filterMinHint = new FilterBase() { @Override + public ReturnCode filterKeyValue(KeyValue ignored) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + + @Override public KeyValue getNextKeyHint(KeyValue currentKV) { return minKeyValue; } @@ -362,6 +367,11 @@ Filter filterMaxHint = new FilterBase() { @Override + public ReturnCode filterKeyValue(KeyValue ignored) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + + @Override public KeyValue getNextKeyHint(KeyValue currentKV) { return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null); } @@ -395,32 +405,36 @@ // MUST PASS ALL - // Should take the max if given two hints + // Should take the first hint filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + minKeyValue)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList(new Filter [] { filterMaxHint, filterMinHint } )); + filterList.filterKeyValue(null); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); - // Should have max hint even if a filter has no hint + // Should have first hint even if a filter has no hint filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList( - new Filter [] { filterMinHint, filterMaxHint, filterNoHint } )); + new Filter [] { filterNoHint, filterMinHint, filterMaxHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - maxKeyValue)); + minKeyValue)); filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(new Filter [] { filterNoHint, filterMinHint } )); + filterList.filterKeyValue(null); assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), minKeyValue)); - - // Should give min hint if its the only one - filterList = new FilterList(Operator.MUST_PASS_ALL, - Arrays.asList(new Filter [] { filterNoHint, filterMinHint } )); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - minKeyValue)); } /** Index: hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java (revision 1507535) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java (working copy) @@ -48,7 +48,6 @@ * *
* Defaults to {@link Operator#MUST_PASS_ALL}. - *

TODO: Fix creation of Configuration on serialization and deserialization. */ @InterfaceAudience.Public @InterfaceStability.Stable @@ -64,6 +63,7 @@ private static final int MAX_LOG_FILTERS = 5; private Operator operator = Operator.MUST_PASS_ALL; private List filters = new ArrayList(); + private Filter seekHintFilter = null; /** Reference KeyValue used by {@link #transform(KeyValue)} for validation purpose. */ private KeyValue referenceKV = null; @@ -163,6 +163,7 @@ for (Filter filter : filters) { filter.reset(); } + seekHintFilter = null; } @Override @@ -232,6 +233,9 @@ case INCLUDE: transformed = filter.transform(transformed); continue; + case SEEK_NEXT_USING_HINT: + seekHintFilter = filter; + return code; default: return code; } @@ -349,22 +353,27 @@ * @return true if and only if the fields of the filter that are serialized * are equal to the corresponding fields in other. Used for testing. */ - boolean areSerializedFieldsEqual(Filter o) { - if (o == this) return true; - if (!(o instanceof FilterList)) return false; + boolean areSerializedFieldsEqual(Filter other) { + if (other == this) return true; + if (!(other instanceof FilterList)) return false; - FilterList other = (FilterList)o; - return this.getOperator().equals(other.getOperator()) && - ((this.getFilters() == other.getFilters()) - || this.getFilters().equals(other.getFilters())); + FilterList o = (FilterList)other; + return this.getOperator().equals(o.getOperator()) && + ((this.getFilters() == o.getFilters()) + || this.getFilters().equals(o.getFilters())); } @Override public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException { KeyValue keyHint = null; + if (operator == Operator.MUST_PASS_ALL) { + keyHint = seekHintFilter.getNextKeyHint(currentKV); + return keyHint; + } + for (Filter filter : filters) { KeyValue curKeyHint = filter.getNextKeyHint(currentKV); - if (curKeyHint == null && operator == Operator.MUST_PASS_ONE) { + if (curKeyHint == null) { // If we ever don't have a hint and this is must-pass-one, then no hint return null; } @@ -374,13 +383,7 @@ keyHint = curKeyHint; continue; } - // There is an existing hint - if (operator == Operator.MUST_PASS_ALL && - KeyValue.COMPARATOR.compare(keyHint, curKeyHint) < 0) { - // If all conditions must pass, we can keep the max hint - keyHint = curKeyHint; - } else if (operator == Operator.MUST_PASS_ONE && - KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) { + if (KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) { // If any condition can pass, we need to keep the min hint keyHint = curKeyHint; }