:100644 100644 86b652c... ee04a43... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java :100644 100644 2a02eec... b0b3535... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java :100644 100644 4e5bac5... bfe311d... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndex.java :100644 100644 d41df0d... e478933... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndexBuilder.java :100644 100644 bb2b2fa... d5eff3f... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/EmptyIndex.java :100644 100644 485ca1d... 9fc5950... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxExpressionEvaluator.java :100644 100644 6b1c645... b8fcb31... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxIndex.java :100644 100644 99a1b1e... 263a01c... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java :100644 100644 fa7b4ba... 7121d43... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/idx/support/sets/BitSet.java :100644 100644 abe2ef9... af49613... M src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestCompleteIndex.java :100644 100644 2e74788... 39df6eb... M src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestIdxExpressionEvaluator.java :100644 100644 60ae619... 15d4c40... M src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestIdxRegion.java :100644 100644 fae65da... 83a9333... M src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/idx/support/sets/IntSetBaseTestCase.java :100644 100644 b6f2353... 13d38eb... M src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/idx/support/sets/TestBitSet.java :100644 100644 2f7f4b0... 65adf0c... M src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/idx/support/sets/TestSparseBitSet.java diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java index 86b652c..ee04a43 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java @@ -34,6 +34,7 @@ public class Comparison extends Expression { private byte[] qualifier; private Operator operator; private byte[] value; + private boolean includeMissing = true; /** * No args constructor. @@ -53,14 +54,46 @@ public class Comparison extends Expression { } /** - * Full constructor with all required fields. + * Convenience constrcutor that takes strings and converts from to byte[]. + * * @param columnName the column name * @param qualifier the column qualifier * @param operator the operator * @param value the value + * @param includeMissing include missing ids + */ + public Comparison(String columnName, String qualifier, Operator operator, + byte[] value, boolean includeMissing) { + this(Bytes.toBytes(columnName), Bytes.toBytes(qualifier), operator, + value, includeMissing); + } + + /** + * Partial constructor with all required fields. + * + * @param columnName the column name + * @param qualifier the column qualifier + * @param operator the operator + * @param value the value + */ + public Comparison(byte[] columnName, byte[] qualifier, Operator operator, + byte[] value) { + this(columnName, qualifier, operator, value, true); + } + + /** + * Full constructor with all fields. + * + * @param columnName the column name + * @param qualifier the column qualifier + * @param operator the operator + * @param value the value + * @param includeMissing should the comparison result include ids which are + * missing from the index. Same idea as {@link org.apache.hadoop.hbase.filter.SingleColumnValueFilter#filterIfMissing}. + * Default value is true. */ public Comparison(byte[] columnName, byte[] qualifier, Operator operator, - byte[] value) { + byte[] value, boolean includeMissing) { assert columnName != null : "The columnName must not be null"; assert qualifier != null : "The qualifier must not be null"; assert operator != null : "The operator must not be null"; @@ -70,6 +103,7 @@ public class Comparison extends Expression { this.qualifier = qualifier; this.operator = operator; this.value = value; + this.includeMissing = includeMissing; } /** @@ -106,6 +140,15 @@ public class Comparison extends Expression { } /** + * Gets whether to include missing columns or not. + * + * @return true to include missing columns. + */ + public boolean getIncludeMissing() { + return includeMissing; + } + + /** * {@inheritDoc} */ @Override @@ -163,24 +206,28 @@ public class Comparison extends Expression { */ public enum Operator { /** - * The equals function. + * The equals operator. */ EQ, /** - * The greater than function. + * The greater than operator. */ GT, /** - * The greater than or equals function. + * The greater than or equals operator. */ GTE, /** - * The less than function. + * The less than operator. */ LT, /** - * The less than or equals function. + * The less than or equals operator. + */ + LTE, + /** + * The not equals operator. */ - LTE + NEQ, } } diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java index 2a02eec..b0b3535 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java @@ -69,13 +69,49 @@ public abstract class Expression implements Writable { /** * Creates and returns an {@link Comparison} * instance. - * @param family the column family name - * @param qualifier the qualifier - * @param operator the operator - * @param value the value + * + * @param family the column family name + * @param qualifier the qualifier + * @param operator the operator + * @param value the value + * @param includeMissing include ids missing from the index. + * Same idea as {@link org.apache.hadoop.hbase.filter.SingleColumnValueFilter#filterIfMissing}. + * true by default + * @return the instance + */ + public static Comparison comparison(byte[] family, byte[] qualifier, Comparison.Operator operator, byte[] value, boolean includeMissing) { + return new Comparison(family, qualifier, operator, value, includeMissing); + } + + /** + * Creates and returns an {@link Comparison} + * instance. + * + * @param family the column family name + * @param qualifier the qualifier + * @param operator the operator + * @param value the value * @return the instance */ public static Comparison comparison(String family, String qualifier, Comparison.Operator operator, byte[] value) { return new Comparison(family, qualifier, operator, value); } + + /** + * Creates and returns an {@link Comparison} + * instance. + * + * @param family the column family name + * @param qualifier the qualifier + * @param operator the operator + * @param value the value + * @param includeMissing include ids missing from the index. + * Same idea as {@link org.apache.hadoop.hbase.filter.SingleColumnValueFilter#filterIfMissing}. + * true by default + * @return the instance + */ + public static Comparison comparison(String family, String qualifier, Comparison.Operator operator, byte[] value, boolean includeMissing) { + return new Comparison(family, qualifier, operator, value, includeMissing); + } + } diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndex.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndex.java index 4e5bac5..bfe311d 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndex.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndex.java @@ -36,39 +36,43 @@ class CompleteIndex implements IdxIndex { * The fixed part in the heap size calcualtion. */ static final long FIXED_SIZE = ClassSize.align(ClassSize.OBJECT + - ClassSize.REFERENCE + 3 * (ClassSize.ARRAY + ClassSize.REFERENCE) + + 2 * ClassSize.REFERENCE + 3 * (ClassSize.ARRAY + ClassSize.REFERENCE) + Bytes.SIZEOF_LONG + 2 * Bytes.SIZEOF_INT ); /** * The capacity of the sets. */ - private int numKeyValues; + private final int numKeyValues; /** * The key store - holds the col:qual values. */ - private List> keyStore; + private final List> keyStore; /** * The value store - holds sets with {@link #numKeyValues} capacity. */ - private IntSet[] valueStore; + private final IntSet[] valueStore; /** * Sets containing partial calculations of the tail operation. */ - private IntSet[] heads; + private final IntSet[] heads; /** * Sets containing partial calculations of the head operation. */ - private IntSet[] tails; + private final IntSet[] tails; + /** + * A set containing all ids matching any key in this index. + */ + private final IntSet allIds; /** * The partial calculation interval (used to determine up to which point * to use the valueStore before grabbing a pre-calculated set. */ - private int precalcInterval; + private final int precalcInterval; /** * The heap size. */ - private long heapSize; + private final long heapSize; /** * Construct a new complete index. @@ -77,20 +81,22 @@ class CompleteIndex implements IdxIndex { * @param valueStore the value store * @param heads a list of precalculated heads * @param tails a list of precalculated tails + * @param allIds a set containing all ids mathcing any key in this index * @param numKeyValues the total number of KeyValues for this region * @param precalcInterval the interval by which tails/heads are precalculated */ CompleteIndex(List> keyStore, IntSet[] valueStore, - IntSet[] heads, IntSet[] tails, + IntSet[] heads, IntSet[] tails, IntSet allIds, int numKeyValues, int precalcInterval) { this.keyStore = keyStore; this.valueStore = valueStore; this.heads = heads; this.tails = tails; + this.allIds = allIds; this.numKeyValues = numKeyValues; this.precalcInterval = precalcInterval; heapSize = FIXED_SIZE + keyStore.heapSize() + calcHeapSize(valueStore) + - calcHeapSize(heads) + calcHeapSize(tails); + calcHeapSize(heads) + calcHeapSize(tails) + allIds.heapSize(); } /** @@ -161,6 +167,16 @@ class CompleteIndex implements IdxIndex { return result; } + /** + * Finds all the results which match any key in this index. + * + * @return all the ids in this index. + */ + @Override + public IntSet all() { + return allIds.clone(); + } + @Override public int size() { return this.keyStore.size(); diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndexBuilder.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndexBuilder.java index d41df0d..e478933 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndexBuilder.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/CompleteIndexBuilder.java @@ -203,15 +203,28 @@ public class CompleteIndexBuilder { IntSet[] heads = new IntSet[precalcSize]; IntSet currentHead = IntSetBuilder.newEmptyIntSet(numKeyValues); + int maxHeadIndex = -1; for (int i = 0; i < indexSize; i++) { currentHead = currentHead.unite(valueStore[i]); if (i % interval == 0) { + maxHeadIndex = i; heads[i / interval] = currentHead; currentHead = currentHead.clone(); } } + + IntSet allIds; + if (maxHeadIndex < 0) { + allIds = IntSetBuilder.newEmptyIntSet(numKeyValues); + } else { + allIds = currentHead.clone(); + // Add all remaning key values to the allKeys set + for (int i = maxHeadIndex; i < indexSize; i++) { + allIds = allIds.unite(valueStore[i]); + } + } - return new CompleteIndex(keyStore, valueStore, heads, tails, + return new CompleteIndex(keyStore, valueStore, heads, tails, allIds, numKeyValues, interval); } else { return new EmptyIndex(keyStore, numKeyValues); diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/EmptyIndex.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/EmptyIndex.java index bb2b2fa..d5eff3f 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/EmptyIndex.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/EmptyIndex.java @@ -81,6 +81,16 @@ public class EmptyIndex implements IdxIndex { /** * {@inheritDoc} + *
+ * Returns an empty set. + */ + @Override + public IntSet all() { + return IntSetBuilder.newEmptyIntSet(numKeyValues); + } + + /** + * {@inheritDoc} */ @Override public String probeToString(byte[] bytes) { diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxExpressionEvaluator.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxExpressionEvaluator.java index 485ca1d..9fc5950 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxExpressionEvaluator.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxExpressionEvaluator.java @@ -98,10 +98,18 @@ public class IdxExpressionEvaluator implements HeapSize { Bytes.toString(comparison.getQualifier()))); IntSet matched = null; + boolean resultIncludesMissing = false; switch (comparison.getOperator()) { case EQ: matched = index.lookup(comparison.getValue()); break; + case NEQ: + matched = index.lookup(comparison.getValue()); + matched = matched.complement(); + // When we complement the matched set we may include ids which are + // missing from the index + resultIncludesMissing = true; + break; case GT: matched = index.tail(comparison.getValue(), false); break; @@ -116,13 +124,19 @@ public class IdxExpressionEvaluator implements HeapSize { break; } + if (comparison.getIncludeMissing() != resultIncludesMissing) { + matched = resultIncludesMissing ? matched.intersect(index.all()) : matched.unite(index.all().complement()); + } + if (LOG.isDebugEnabled() && matched != null) { LOG.debug(String.format("Evaluation of comparison on column: '%s', " + - "qualifier: '%s', operator: %s, value: '%s' yielded %s matches", - Bytes.toString(comparison.getColumnName()), - Bytes.toString(comparison.getQualifier()), - comparison.getOperator(), - index.probeToString(comparison.getValue()), matched.size())); + "qualifier: '%s', operator: %s, value: '%s' include missing: '%b' " + + "yielded %s matches", + Bytes.toString(comparison.getColumnName()), + Bytes.toString(comparison.getQualifier()), + comparison.getOperator(), + index.probeToString(comparison.getValue()), + comparison.getIncludeMissing(), matched.size())); } return matched != null ? matched : null; diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxIndex.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxIndex.java index 6b1c645..b8fcb31 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxIndex.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxIndex.java @@ -55,6 +55,13 @@ public interface IdxIndex extends HeapSize { IntSet head(byte[] probe, boolean inclusive); /** + * Finds all the results which match any key in this index. + * + * @return all the ids in this index. + */ + IntSet all(); + + /** * Returns a string representation of the provided bytes probe. * * @param bytes the bytes diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java index 99a1b1e..263a01c 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java @@ -388,7 +388,7 @@ public class IdxRegion extends HRegion { for (byte[] family : regionInfo.getTableDesc().getFamiliesKeys()) { Store store = stores.get(family); scanners.addAll(getMemstoreScanners(store, scan.getStartRow())); - break; // we only need one + //break; // we only need one } return new KeyValueHeap(scanners.toArray(new KeyValueScanner[scanners.size()]), comparator); } diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/idx/support/sets/BitSet.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/idx/support/sets/BitSet.java index fa7b4ba..7121d43 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/idx/support/sets/BitSet.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/idx/support/sets/BitSet.java @@ -177,12 +177,14 @@ class BitSet extends IntSetBase { */ @Override public IntSet complement() { - size = -1; - for (int i = 0; i < words.length; i++) { - words[i] = ~words[i]; + if (capacity > 0) { + size = -1; + for (int i = 0; i < words.length; i++) { + words[i] = ~words[i]; + } + words[words.length - 1] ^= capacity % WORD_BIT_COUNT == 0 ? + 0 : -1L << capacity; // get rid of the trailing ones } - words[words.length - 1] ^= capacity % WORD_BIT_COUNT == 0 ? - 0 : -1L << capacity; // get rid of the trailing ones return this; } diff --git a/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestCompleteIndex.java b/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestCompleteIndex.java index abe2ef9..af49613 100644 --- a/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestCompleteIndex.java +++ b/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestCompleteIndex.java @@ -316,17 +316,39 @@ public class TestCompleteIndex extends HBaseTestCase { return TestBitSet.createBitSet(NUM_KEY_VALUES, items); } - private static CompleteIndex fillIndex(long[] values, int[] ids) { - Assert.assertEquals(values.length, ids.length); - HColumnDescriptor columnDescriptor = new HColumnDescriptor(FAMILY); + private static CompleteIndex fillIndex(long[] keys, int[] ids) { + return fillIndex(keys, ids, FAMILY, QUALIFIER, NUM_KEY_VALUES); + } + + /** + * A utility method to create a complete index. + * + * @param keys the index keys + * @param ids the key/value ids + * @param family the family id + * @param qualifier the qualifier id + * @param numKeyValues the total number of key/values when finalizing the + * index + * @return a new, populated index. + */ + static CompleteIndex fillIndex(long[] keys, int[] ids, byte[] family, + byte[] qualifier, int numKeyValues) { + Assert.assertEquals(keys.length, ids.length); + HColumnDescriptor columnDescriptor = new HColumnDescriptor(family); CompleteIndexBuilder completeIndex = new CompleteIndexBuilder(columnDescriptor, - new IdxIndexDescriptor(QUALIFIER, IdxQualifierType.LONG)); - for (int i = 0; i < values.length; i++) { - completeIndex.addKeyValue(new KeyValue(Bytes.toBytes(ids[i]), FAMILY, - QUALIFIER, Bytes.toBytes(values[i])), ids[i]); + new IdxIndexDescriptor(qualifier, IdxQualifierType.LONG)); + for (int i = 0; i < keys.length; i++) { + completeIndex.addKeyValue(new KeyValue(Bytes.toBytes(ids[i]), family, + qualifier, Bytes.toBytes(keys[i])), ids[i]); + } + CompleteIndex ix = (CompleteIndex) completeIndex.finalizeIndex(numKeyValues); + IntSet allIds = ix.all(); + Assert.assertEquals(ids.length, allIds.size()); + for (int id : ids){ + Assert.assertTrue(allIds.contains(id)); } - return (CompleteIndex) completeIndex.finalizeIndex(NUM_KEY_VALUES); + return ix; } } diff --git a/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestIdxExpressionEvaluator.java b/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestIdxExpressionEvaluator.java index 2e74788..39df6eb 100644 --- a/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestIdxExpressionEvaluator.java +++ b/src/contrib/indexed/src/test/org/apache/hadoop/hbase/regionserver/TestIdxExpressionEvaluator.java @@ -19,25 +19,32 @@ */ package org.apache.hadoop.hbase.regionserver; -import junit.framework.TestCase; import junit.framework.Assert; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.Bytes; +import junit.framework.TestCase; +import org.apache.hadoop.hbase.client.idx.exp.Comparison; +import org.apache.hadoop.hbase.client.idx.exp.Expression; import org.apache.hadoop.hbase.regionserver.idx.support.sets.IntSet; -//import org.apache.hadoop.hbase.regionserver.idx.support.sets.BitSet; import org.apache.hadoop.hbase.regionserver.idx.support.sets.IntSetBuilder; -import org.apache.hadoop.hbase.client.idx.exp.Expression; -import org.apache.hadoop.hbase.client.idx.exp.Comparison; +import org.apache.hadoop.hbase.regionserver.idx.support.sets.TestBitSet; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.easymock.EasyMock; -import java.util.Map; import java.util.HashMap; +import java.util.Map; /** * Tests the {@link IdxExpressionEvaluator} class. */ public class TestIdxExpressionEvaluator extends TestCase { + + private static final int NUM_KEY_VALUES = 107; + private static final byte[] FAMILY = Bytes.toBytes("f"); + private static final byte[] QUALIFIER = Bytes.toBytes("q"); + private TestSearchContext searchContext; + private static final long[] KEYS = new long[]{1, 4, 9, 7, 1, 2, 8, 1, 4, 1, 9}; + private static final int[] IDS = new int[]{1, 7, 8, 32, 40, 66, 67, 80, 86, 90, 106}; @Override protected void setUp() throws Exception { @@ -60,7 +67,7 @@ public class TestIdxExpressionEvaluator extends TestCase { // perform the test IdxExpressionEvaluator evaluator = new IdxExpressionEvaluator(); - Expression exp = Expression.comparison(column, qualifier, Comparison.Operator.EQ, value); + Expression exp = Expression.comparison(column, qualifier, Comparison.Operator.EQ, value, false); IntSet intSet = evaluator.evaluate(searchContext, exp); // assert the evaluator interacted with the indices correctly @@ -69,6 +76,98 @@ public class TestIdxExpressionEvaluator extends TestCase { } /** + * Tests the neq op. + */ + public void testNEQ() { + IntSet intSet = evaluateComprison(Comparison.Operator.NEQ, 1L, false); + assertSetsEqual(makeSet(7, 8, 32, 66, 67, 86, 106), intSet); + + intSet = evaluateComprison(Comparison.Operator.NEQ, 1L, true); + assertSetsEqual(makeSet(IDS).complement().unite(makeSet(7, 8, 32, 66, 67, 86, 106)), intSet); + } + + /** + * Tests the eq op. + */ + public void testEQ() { + IntSet intSet = evaluateComprison(Comparison.Operator.EQ, 4L, false); + assertSetsEqual(makeSet(7, 86), intSet); + + intSet = evaluateComprison(Comparison.Operator.EQ, 4L, true); + assertSetsEqual(makeSet(IDS).complement().unite(makeSet(7, 86)), intSet); + } + + /** + * Tests the gt op. + */ + public void testGT() { + IntSet intSet = evaluateComprison(Comparison.Operator.GT, 7L, false); + assertSetsEqual(makeSet(8, 67, 106), intSet); + + intSet = evaluateComprison(Comparison.Operator.GT, 7L, true); + assertSetsEqual(makeSet(IDS).complement().unite(makeSet(8, 67, 106)), intSet); + + } + + /** + * Tests the gt op. + */ + public void testGTE() { + IntSet intSet = evaluateComprison(Comparison.Operator.GTE, 8L, false); + assertSetsEqual(makeSet(8, 67, 106), intSet); + + intSet = evaluateComprison(Comparison.Operator.GTE, 8L, true); + assertSetsEqual(makeSet(IDS).complement().unite(makeSet(8, 67, 106)), intSet); + } + + /** + * Tests the gt op. + */ + public void testLT() { + IntSet intSet = evaluateComprison(Comparison.Operator.LT, 3L, false); + assertSetsEqual(makeSet(1, 40, 66, 80, 90), intSet); + + intSet = evaluateComprison(Comparison.Operator.LT, 3L, true); + assertSetsEqual(makeSet(IDS).complement().unite(makeSet(1, 40, 66, 80, 90)), intSet); + + } + + /** + * Tests the gt op. + */ + public void testLTE() { + IntSet intSet = evaluateComprison(Comparison.Operator.LTE, 2L, false); + assertSetsEqual(makeSet(1, 40, 66, 80, 90), intSet); + + intSet = evaluateComprison(Comparison.Operator.LTE, 2L, true); + assertSetsEqual(makeSet(IDS).complement().unite(makeSet(1, 40, 66, 80, 90)), intSet); + } + + private IntSet evaluateComprison(Comparison.Operator op, long value, boolean includeMissing) { + IdxIndex index = TestCompleteIndex.fillIndex(KEYS, IDS, FAMILY, QUALIFIER, NUM_KEY_VALUES); + TestSearchContext sc = new TestSearchContext(); + sc.indices.put(Pair.of(FAMILY, QUALIFIER), index); + + IdxExpressionEvaluator evaluator = new IdxExpressionEvaluator(); + return evaluator.evaluate(sc, + Expression.comparison(FAMILY, QUALIFIER, op, Bytes.toBytes(value), includeMissing)); + } + + private static void assertSetsEqual(IntSet is1, IntSet is2) { + Assert.assertEquals(is1.capacity(), is2.capacity()); + Assert.assertEquals(is1.size(), is2.size()); + IntSet.IntSetIterator iter = is1.iterator(); + while (iter.hasNext()) { + int element = iter.next(); + Assert.assertTrue("element: " + element, is2.contains(element)); + } + } + + private IntSet makeSet(int... elements) { + return TestBitSet.createBitSet(NUM_KEY_VALUES, elements); + } + + /** * Tests an Or expression containing two comparisons. */ public void testEvaluationOrExpression() { @@ -77,7 +176,7 @@ public class TestIdxExpressionEvaluator extends TestCase { byte[] qualifier1 = Bytes.toBytes("qualifier1"); byte[] value1 = Bytes.toBytes("value1"); IdxIndex index1 = EasyMock.createMock(IdxIndex.class); - IntSet bitSet1 = new IntSetBuilder().start().addAll(1,2,3,4,5).finish(100); + IntSet bitSet1 = new IntSetBuilder().start().addAll(1, 2, 3, 4, 5).finish(100); EasyMock.expect(index1.head(value1, false)).andReturn(bitSet1); EasyMock.expect(index1.probeToString(value1)).andReturn(Bytes.toString(value1)).anyTimes(); EasyMock.replay(index1); @@ -96,8 +195,8 @@ public class TestIdxExpressionEvaluator extends TestCase { // perform the test IdxExpressionEvaluator evaluator = new IdxExpressionEvaluator(); Expression exp = Expression.or( - Expression.comparison(column1, qualifier1, Comparison.Operator.LT, value1), - Expression.comparison(column2, qualifier2, Comparison.Operator.GT, value2) + Expression.comparison(column1, qualifier1, Comparison.Operator.LT, value1, false), + Expression.comparison(column2, qualifier2, Comparison.Operator.GT, value2, false) ); IntSet intSet = evaluator.evaluate(searchContext, exp); @@ -137,8 +236,8 @@ public class TestIdxExpressionEvaluator extends TestCase { // perform the test IdxExpressionEvaluator evaluator = new IdxExpressionEvaluator(); Expression exp = Expression.and( - Expression.comparison(column1, qualifier1, Comparison.Operator.LTE, value1), - Expression.comparison(column2, qualifier2, Comparison.Operator.GTE, value2) + Expression.comparison(column1, qualifier1, Comparison.Operator.LTE, value1, false), + Expression.comparison(column2, qualifier2, Comparison.Operator.GTE, value2, false) ); IntSet intSet = evaluator.evaluate(searchContext, exp); @@ -160,7 +259,7 @@ public class TestIdxExpressionEvaluator extends TestCase { byte[] column1 = Bytes.toBytes("column1"); byte[] qualifier1 = Bytes.toBytes("qualifier1"); byte[] value1 = Bytes.toBytes("value1"); - IntSet bitSet1 = new IntSetBuilder().start().addAll(1,2,3,4,5,6).finish(100); + IntSet bitSet1 = new IntSetBuilder().start().addAll(1, 2, 3, 4, 5, 6).finish(100); EasyMock.expect(index1.head(value1, true)).andReturn(bitSet1); EasyMock.expect(index1.probeToString(value1)).andReturn(Bytes.toString(value1)).anyTimes(); EasyMock.replay(index1); @@ -189,11 +288,11 @@ public class TestIdxExpressionEvaluator extends TestCase { // perform the test IdxExpressionEvaluator evaluator = new IdxExpressionEvaluator(); Expression exp = Expression.or( - Expression.and( - Expression.comparison(column1, qualifier1, Comparison.Operator.LTE, value1), - Expression.comparison(column2, qualifier2, Comparison.Operator.GTE, value2) - ), - Expression.comparison(column3, qualifier3, Comparison.Operator.EQ, value3) + Expression.and( + Expression.comparison(column1, qualifier1, Comparison.Operator.LTE, value1, false), + Expression.comparison(column2, qualifier2, Comparison.Operator.GTE, value2, false) + ), + Expression.comparison(column3, qualifier3, Comparison.Operator.EQ, value3, false) ); IntSet intSet = evaluator.evaluate(searchContext, exp); @@ -243,7 +342,7 @@ public class TestIdxExpressionEvaluator extends TestCase { public TestSearchContext() { super(null, null); - indices = new HashMap