:100644 100644 1da9cf8... 0edd56d... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/IdxScan.java :100644 100644 08d2a97... 027e873... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/And.java :100644 100644 ee04a43... 563a43f... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java :100644 100644 aac9837... fd425b7... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Compound.java :100644 100644 b0b3535... 6c0c545... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java :100644 100644 f6330e6... 722482a... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Or.java :100644 100644 263a01c... 32d39b4... M src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java :100644 100644 9e14c3a... ccf3edb... M src/contrib/indexed/src/test/org/apache/hadoop/hbase/client/idx/exp/TestExpression.java diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/IdxScan.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/IdxScan.java index 1da9cf8..0edd56d 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/IdxScan.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/IdxScan.java @@ -20,11 +20,11 @@ package org.apache.hadoop.hbase.client.idx; import org.apache.hadoop.hbase.WritableHelper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.client.idx.exp.Expression; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; @@ -32,7 +32,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.Map; -import java.util.Collection; /** * Extends the {@link Scan} class to provide an {@link Expression} @@ -176,19 +175,26 @@ public class IdxScan extends Scan { } public static Expression getExpression(Scan scan) throws IOException { - if (scan instanceof IdxScan && ((IdxScan) scan).getExpression() != null) { - return ((IdxScan) scan).getExpression(); + if (scan == null) { + return null; } - Map values = scan.getValues(); - if (values.containsKey(EXPRESSION)) { - DataInputBuffer in = new DataInputBuffer(); - byte[] bytes = values.get(EXPRESSION).get(); - in.reset(bytes, bytes.length); - - return WritableHelper.readInstanceNullable(in, Expression.class); + Expression expression; + if (scan instanceof IdxScan && ((IdxScan) scan).getExpression() != null) { + expression = ((IdxScan) scan).getExpression(); } else { - return null; + Map values = scan.getValues(); + if (values.containsKey(EXPRESSION)) { + DataInputBuffer in = new DataInputBuffer(); + byte[] bytes = values.get(EXPRESSION).get(); + in.reset(bytes, bytes.length); + + expression = WritableHelper.readInstanceNullable(in, Expression.class); + } else { + return null; + } } + + return expression; } } diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/And.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/And.java index 08d2a97..027e873 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/And.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/And.java @@ -19,6 +19,8 @@ */ package org.apache.hadoop.hbase.client.idx.exp; +import org.apache.hadoop.hbase.filter.FilterList; + import java.util.Collection; /** @@ -50,7 +52,16 @@ public class And extends Compound { } /** - * Adds the expression to the set of expression. + * {@inheritDoc}. + */ + @Override + protected FilterList.Operator getFilterOperator() { + return FilterList.Operator.MUST_PASS_ALL; + } + + /** + * Adds the expression to the set of expression. If the expression is null + * then no action is performed. * @param expression the expression * @return this * @see Compound#add(Expression) diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java index ee04a43..563a43f 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Comparison.java @@ -19,6 +19,9 @@ */ package org.apache.hadoop.hbase.client.idx.exp; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.util.Bytes; import java.io.DataInput; @@ -192,6 +195,41 @@ public class Comparison extends Expression { * {@inheritDoc} */ @Override + public Filter toFilter() { + CompareFilter.CompareOp op = null; + switch (getOperator()) { + case EQ: + op = CompareFilter.CompareOp.EQUAL; + break; + case GT: + op = CompareFilter.CompareOp.GREATER; + break; + case GTE: + op = CompareFilter.CompareOp.GREATER_OR_EQUAL; + break; + case LT: + op = CompareFilter.CompareOp.LESS; + break; + case LTE: + op = CompareFilter.CompareOp.LESS_OR_EQUAL; + break; + case NEQ: + op = CompareFilter.CompareOp.NOT_EQUAL; + break; + } + + return new SingleColumnValueFilter( + getColumnName(), + getQualifier(), + op, + getValue() + ); + } + + /** + * {@inheritDoc} + */ + @Override public int hashCode() { int result = Arrays.hashCode(columnName); result = 31 * result + Arrays.hashCode(qualifier); diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Compound.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Compound.java index aac9837..fd425b7 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Compound.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Compound.java @@ -21,13 +21,15 @@ package org.apache.hadoop.hbase.client.idx.exp; import org.apache.hadoop.hbase.WritableHelper; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -45,8 +47,10 @@ public abstract class Compound extends Expression { * @param expressions the expressions to be evaluated */ public Compound(Expression... expressions) { - assert expressions != null : "expressions cannot be null or empty"; - this.children = new HashSet(Arrays.asList(expressions)); + this.children = new LinkedHashSet(); + for (Expression expression : expressions) { + add(expression); + } } /** @@ -58,12 +62,15 @@ public abstract class Compound extends Expression { } /** - * Add an expression to the child set. + * Add an expression to the child set. If the expression is null then to action + * is performed. * @param expression the expression to add * @return this */ public Compound add(Expression expression) { - this.children.add(expression); + if (expression != null) { + this.children.add(expression); + } return this; } @@ -76,6 +83,28 @@ public abstract class Compound extends Expression { } /** + * {@inheritDoc}. + * + * This impl. returns a {@link org.apache.hadoop.hbase.filter.FilterList} which + * includes all the children. + */ + @Override + public Filter toFilter() { + FilterList filterList = new FilterList(getFilterOperator()); + for (Expression expression : getChildren()) { + filterList.addFilter(expression.toFilter()); + } + return filterList; + } + + /** + * Returns the filter operator to use when creating a filter for this compound + * expression. + * @return the filter operator + */ + protected abstract FilterList.Operator getFilterOperator(); + + /** * {@inheritDoc} */ @Override diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java index b0b3535..6c0c545 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Expression.java @@ -19,6 +19,7 @@ */ package org.apache.hadoop.hbase.client.idx.exp; +import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.io.Writable; /** @@ -26,6 +27,12 @@ import org.apache.hadoop.io.Writable; */ public abstract class Expression implements Writable { /** + * Converts the expression to a filter. + * @return the expression as a filter + */ + public abstract Filter toFilter(); + + /** * {@inheritDoc} */ public abstract int hashCode(); diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Or.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Or.java index f6330e6..722482a 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Or.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/client/idx/exp/Or.java @@ -19,6 +19,8 @@ */ package org.apache.hadoop.hbase.client.idx.exp; +import org.apache.hadoop.hbase.filter.FilterList; + import java.util.Collection; /** @@ -50,7 +52,16 @@ public class Or extends Compound { } /** - * Adds the expression to the set of expression. + * {@inheritDoc}. + */ + @Override + protected FilterList.Operator getFilterOperator() { + return FilterList.Operator.MUST_PASS_ONE; + } + + /** + * Adds the expression to the set of expression. If the expression is null + * then no action is performed. * @param expression the expression * @return this * @see Compound#add(Expression) diff --git a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java index 263a01c..32d39b4 100644 --- a/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java +++ b/src/contrib/indexed/src/java/org/apache/hadoop/hbase/regionserver/IdxRegion.java @@ -23,11 +23,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.JmxHelper; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.idx.IdxScan; import org.apache.hadoop.hbase.client.idx.exp.Expression; @@ -147,7 +147,7 @@ public class IdxRegion extends HRegion { protected InternalScanner instantiateInternalScanner(Scan scan, List additionalScanners) throws IOException { Expression expression = IdxScan.getExpression(scan); - if (scan == null || expression == null) { + if (expression == null) { totalNonIndexedScans.incrementAndGet(); return super.instantiateInternalScanner(scan, additionalScanners); } else { @@ -286,7 +286,7 @@ public class IdxRegion extends HRegion { private KeyValue lastKeyValue; IdxRegionScanner(Scan scan, IdxSearchContext idxSearchContext, - IntSet matchedExpression) throws IOException { + IntSet matchedExpression) throws IOException { super(scan); numberOfOngoingIndexedScans.incrementAndGet(); keyProvider = new KeyProvider(idxSearchContext, matchedExpression, scan); diff --git a/src/contrib/indexed/src/test/org/apache/hadoop/hbase/client/idx/exp/TestExpression.java b/src/contrib/indexed/src/test/org/apache/hadoop/hbase/client/idx/exp/TestExpression.java index 9e14c3a..ccf3edb 100644 --- a/src/contrib/indexed/src/test/org/apache/hadoop/hbase/client/idx/exp/TestExpression.java +++ b/src/contrib/indexed/src/test/org/apache/hadoop/hbase/client/idx/exp/TestExpression.java @@ -21,11 +21,10 @@ package org.apache.hadoop.hbase.client.idx.exp; import junit.framework.Assert; import junit.framework.TestCase; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.client.idx.exp.And; -import org.apache.hadoop.hbase.client.idx.exp.Comparison; -import org.apache.hadoop.hbase.client.idx.exp.Expression; -import org.apache.hadoop.hbase.client.idx.exp.Or; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; @@ -85,6 +84,47 @@ public class TestExpression extends TestCase { } /** + * Tests the ability to convert an expression to a filter. + */ + public void testExpressionToFilter() { + String columnName1 = "columnName1"; + String qualifer1 = "qualifier1"; + byte[] value1 = Bytes.toBytes("value1"); + Comparison.Operator operator1 = Comparison.Operator.EQ; + + String columnName2 = "columnName2"; + String qualifer2 = "qualifier2"; + byte[] value2 = Bytes.toBytes("value2"); + Comparison.Operator operator2 = Comparison.Operator.GT; + + String columnName3 = "columnName3"; + String qualifer3 = "qualifier3"; + byte[] value3 = Bytes.toBytes("value3"); + Comparison.Operator operator3 = Comparison.Operator.LT; + + Expression expression = Expression + .or( + Expression.comparison(columnName1, qualifer1, operator1, value1) + ) + .or( + Expression.and() + .and(Expression.comparison(columnName2, qualifer2, operator2, value2)) + .and(Expression.comparison(columnName3, qualifer3, operator3, value3)) + ); + + Filter filter = expression.toFilter(); + + Assert.assertEquals("The filter was of the wrong type", FilterList.class, filter.getClass()); + Assert.assertEquals("The filter list operator was of the wrong type", FilterList.Operator.MUST_PASS_ONE, ((FilterList) filter).getOperator()); + Assert.assertEquals("The filter list had the wrong number of filters", 2, ((FilterList) filter).getFilters().size()); + Assert.assertEquals("The filter was of the wrong type", SingleColumnValueFilter.class, ((FilterList) filter).getFilters().get(0).getClass()); + Assert.assertEquals("The filter was of the wrong type", FilterList.class, ((FilterList) filter).getFilters().get(1).getClass()); + FilterList internalFilterList = (FilterList) ((FilterList) filter).getFilters().get(1); + Assert.assertEquals("The filter was of the wrong type", SingleColumnValueFilter.class, internalFilterList.getFilters().get(0).getClass()); + Assert.assertEquals("The filter was of the wrong type", SingleColumnValueFilter.class, internalFilterList.getFilters().get(1).getClass()); + } + + /** * Tests the an expression tree can be written and read and still be equal. * * @throws java.io.IOException if an io error occurs