From 058b106778b37e4a5b250c84b6313afeb4770874 Mon Sep 17 00:00:00 2001 From: huzheng Date: Thu, 19 Oct 2017 21:51:18 +0800 Subject: [PATCH] HBASE-18993 Backport patches in HBASE-18410 to branch-1.x branches --- .../org/apache/hadoop/hbase/filter/Filter.java | 14 +- .../org/apache/hadoop/hbase/filter/FilterList.java | 407 ++++---------- .../apache/hadoop/hbase/filter/FilterListBase.java | 172 ++++++ .../hadoop/hbase/filter/FilterListWithAND.java | 292 ++++++++++ .../hadoop/hbase/filter/FilterListWithOR.java | 424 ++++++++++++++ .../apache/hadoop/hbase/filter/TestFilterList.java | 616 +++++++++++++++------ .../hadoop/hbase/filter/TestFilterListOnMini.java | 6 +- 7 files changed, 1466 insertions(+), 465 deletions(-) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java index 0a7a184..a22b5c7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java @@ -166,15 +166,23 @@ public abstract class Filter { */ NEXT_COL, /** - * Done with columns, skip to next row. Note that filterRow() will - * still be called. + * Seek to next row in current family. It may still pass a cell whose family is different but + * row is the same as previous cell to {@link #filterKeyValue(Cell)} , even if we get a NEXT_ROW + * returned for previous cell. For more details see HBASE-18368.
+ * Once reset() method was invoked, then we switch to the next row for all family, and you can + * catch the event by invoking CellUtils.matchingRows(previousCell, currentCell).
+ * Note that filterRow() will still be called.
*/ NEXT_ROW, /** * Seek to next key which is given as hint by the filter. */ SEEK_NEXT_USING_HINT, -} + /** + * Include KeyValue and done with row, seek to next. See NEXT_ROW + */ + INCLUDE_AND_SEEK_NEXT_ROW, + } /** * Chance to alter the list of Cells to be submitted. Modifications to the list will carry on diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java index be22e5d..1b1b61a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.filter; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -35,109 +36,84 @@ import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import com.google.protobuf.InvalidProtocolBufferException; /** - * Implementation of {@link Filter} that represents an ordered List of Filters - * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL} - * (AND) or {@link Operator#MUST_PASS_ONE} (OR). - * Since you can use Filter Lists as children of Filter Lists, you can create a - * hierarchy of filters to be evaluated. - * - *
- * {@link Operator#MUST_PASS_ALL} evaluates lazily: evaluation stops as soon as one filter does - * not include the KeyValue. - * - *
- * {@link Operator#MUST_PASS_ONE} evaluates non-lazily: all filters are always evaluated. - * - *
+ * Implementation of {@link Filter} that represents an ordered List of Filters which will be + * evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL} (AND) or + * {@link Operator#MUST_PASS_ONE} (OR). Since you can use Filter Lists as children of + * Filter Lists, you can create a hierarchy of filters to be evaluated.
+ * {@link Operator#MUST_PASS_ALL} evaluates lazily: evaluation stops as soon as one filter does not + * include the KeyValue.
+ * {@link Operator#MUST_PASS_ONE} evaluates non-lazily: all filters are always evaluated.
* Defaults to {@link Operator#MUST_PASS_ALL}. */ @InterfaceAudience.Public @InterfaceStability.Stable final public class FilterList extends Filter { + /** set operator */ @InterfaceAudience.Public @InterfaceStability.Stable - public static enum Operator { + public enum Operator { /** !AND */ MUST_PASS_ALL, /** !OR */ MUST_PASS_ONE } - private static final int MAX_LOG_FILTERS = 5; private Operator operator = Operator.MUST_PASS_ALL; - private List filters = new ArrayList(); - private Filter seekHintFilter = null; - - /** Reference Cell used by {@link #transformCell(Cell)} for validation purpose. */ - private Cell referenceKV = null; - - /** - * When filtering a given Cell in {@link #filterKeyValue(Cell)}, - * this stores the transformed Cell to be returned by {@link #transformCell(Cell)}. - * - * Individual filters transformation are applied only when the filter includes the Cell. - * Transformations are composed in the order specified by {@link #filters}. - */ - private Cell transformedKV = null; + private FilterListBase filterListBase; /** - * Constructor that takes a set of {@link Filter}s. The default operator - * MUST_PASS_ALL is assumed. - * - * @param rowFilters list of filters + * Constructor that takes a set of {@link Filter}s and an operator. + * @param operator Operator to process filter set with. + * @param filters Set of row filters. */ - public FilterList(final List rowFilters) { - if (rowFilters instanceof ArrayList) { - this.filters = rowFilters; + public FilterList(final Operator operator, final List filters) { + if (operator == Operator.MUST_PASS_ALL) { + filterListBase = new FilterListWithAND(filters); + } else if (operator == Operator.MUST_PASS_ONE) { + filterListBase = new FilterListWithOR(filters); } else { - this.filters = new ArrayList(rowFilters); + throw new IllegalArgumentException("Invalid operator: " + operator); } + this.operator = operator; } /** - * Constructor that takes a var arg number of {@link Filter}s. The fefault operator - * MUST_PASS_ALL is assumed. - * @param rowFilters + * Constructor that takes a set of {@link Filter}s. The default operator MUST_PASS_ALL is assumed. + * @param filters list of filters */ - public FilterList(final Filter... rowFilters) { - this.filters = new ArrayList(Arrays.asList(rowFilters)); + public FilterList(final List filters) { + this(Operator.MUST_PASS_ALL, filters); } /** - * Constructor that takes an operator. - * - * @param operator Operator to process filter set with. + * Constructor that takes a var arg number of {@link Filter}s. The default operator MUST_PASS_ALL + * is assumed. + * @param filters */ - public FilterList(final Operator operator) { - this.operator = operator; + public FilterList(final Filter... filters) { + this(Operator.MUST_PASS_ALL, Arrays.asList(filters)); } /** - * Constructor that takes a set of {@link Filter}s and an operator. - * + * Constructor that takes an operator. * @param operator Operator to process filter set with. - * @param rowFilters Set of row filters. */ - public FilterList(final Operator operator, final List rowFilters) { - this.filters = new ArrayList(rowFilters); - this.operator = operator; + public FilterList(final Operator operator) { + this(operator, new ArrayList()); } /** * Constructor that takes a var arg number of {@link Filter}s and an operator. - * * @param operator Operator to process filter set with. - * @param rowFilters Filters to use + * @param filters Filters to use */ - public FilterList(final Operator operator, final Filter... rowFilters) { - this.filters = new ArrayList(Arrays.asList(rowFilters)); - this.operator = operator; + public FilterList(final Operator operator, final Filter... filters) { + this(operator, Arrays.asList(filters)); } /** * Get the operator. - * * @return operator */ public Operator getOperator() { @@ -146,245 +122,110 @@ final public class FilterList extends Filter { /** * Get the filters. - * * @return filters */ public List getFilters() { - return filters; + return filterListBase.getFilters(); + } + + public int size() { + return this.filterListBase.size(); + } + + public void addFilter(List filters) { + filterListBase.addFilterLists(filters); } /** * Add a filter. - * * @param filter another filter */ public void addFilter(Filter filter) { - if (this.isReversed() != filter.isReversed()) { - throw new IllegalArgumentException( - "Filters in the list must have the same reversed flag, this.reversed=" - + this.isReversed()); - } - this.filters.add(filter); + addFilter(Collections.singletonList(filter)); } @Override public void reset() throws IOException { - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - filters.get(i).reset(); - } - seekHintFilter = null; + filterListBase.reset(); } @Override public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException { - boolean flag = (this.operator == Operator.MUST_PASS_ONE) ? true : false; - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - Filter filter = filters.get(i); - if (this.operator == Operator.MUST_PASS_ALL) { - if (filter.filterAllRemaining() || - filter.filterRowKey(rowKey, offset, length)) { - flag = true; - } - } else if (this.operator == Operator.MUST_PASS_ONE) { - if (!filter.filterAllRemaining() && - !filter.filterRowKey(rowKey, offset, length)) { - flag = false; - } - } - } - return flag; + return filterListBase.filterRowKey(rowKey, offset, length); } @Override public boolean filterAllRemaining() throws IOException { - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - if (filters.get(i).filterAllRemaining()) { - if (operator == Operator.MUST_PASS_ALL) { - return true; - } - } else { - if (operator == Operator.MUST_PASS_ONE) { - return false; - } - } - } - return operator == Operator.MUST_PASS_ONE; + return filterListBase.filterAllRemaining(); } @Override - public Cell transformCell(Cell v) throws IOException { - // transformCell() is expected to follow an inclusive filterKeyValue() immediately: - if (!v.equals(this.referenceKV)) { - throw new IllegalStateException("Reference Cell: " + this.referenceKV + " does not match: " - + v); - } - return this.transformedKV; + public Cell transformCell(Cell c) throws IOException { + return filterListBase.transformCell(c); } /** - * WARNING: please to not override this method. Instead override {@link #transformCell(Cell)}. - * - * When removing this, its body should be placed in transformCell. - * - * This is for transition from 0.94 -> 0.96 + * WARNING: please to not override this method. Instead override {@link #transformCell(Cell)}. + * When removing this, its body should be placed in transformCell. This is for transition from + * 0.94 -> 0.96 */ @Deprecated @Override public KeyValue transform(KeyValue v) throws IOException { - // transform() is expected to follow an inclusive filterKeyValue() immediately: - if (!v.equals(this.referenceKV)) { - throw new IllegalStateException( - "Reference Cell: " + this.referenceKV + " does not match: " + v); - } - return KeyValueUtil.ensureKeyValue(this.transformedKV); + return KeyValueUtil.ensureKeyValue(transformCell((Cell) v)); } - - @Override - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SF_SWITCH_FALLTHROUGH", - justification="Intentional") - public ReturnCode filterKeyValue(Cell v) throws IOException { - this.referenceKV = v; - - // Accumulates successive transformation of every filter that includes the Cell: - Cell transformed = v; - - ReturnCode rc = operator == Operator.MUST_PASS_ONE? - ReturnCode.SKIP: ReturnCode.INCLUDE; - int listize = filters.size(); - /* - * When all filters in a MUST_PASS_ONE FilterList return a SEEK_USING_NEXT_HINT code, - * we should return SEEK_NEXT_USING_HINT from the FilterList to utilize the lowest seek value. - * - * The following variable tracks whether any of the Filters returns ReturnCode other than - * SEEK_NEXT_USING_HINT for MUST_PASS_ONE FilterList, in which case the optimization would - * be skipped. - */ - boolean seenNonHintReturnCode = false; - for (int i = 0; i < listize; i++) { - Filter filter = filters.get(i); - if (operator == Operator.MUST_PASS_ALL) { - if (filter.filterAllRemaining()) { - return ReturnCode.NEXT_ROW; - } - ReturnCode code = filter.filterKeyValue(v); - switch (code) { - // Override INCLUDE and continue to evaluate. - case INCLUDE_AND_NEXT_COL: - rc = ReturnCode.INCLUDE_AND_NEXT_COL; // FindBugs SF_SWITCH_FALLTHROUGH - case INCLUDE: - transformed = filter.transformCell(transformed); - continue; - case SEEK_NEXT_USING_HINT: - seekHintFilter = filter; - return code; - default: - return code; - } - } else if (operator == Operator.MUST_PASS_ONE) { - if (filter.filterAllRemaining()) { - seenNonHintReturnCode = true; - continue; - } - - ReturnCode localRC = filter.filterKeyValue(v); - if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) { - seenNonHintReturnCode = true; - } - switch (localRC) { - case INCLUDE: - if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) { - rc = ReturnCode.INCLUDE; - } - transformed = filter.transformCell(transformed); - break; - case INCLUDE_AND_NEXT_COL: - rc = ReturnCode.INCLUDE_AND_NEXT_COL; - transformed = filter.transformCell(transformed); - // must continue here to evaluate all filters - break; - case NEXT_ROW: - break; - case SKIP: - break; - case NEXT_COL: - break; - case SEEK_NEXT_USING_HINT: - break; - default: - throw new IllegalStateException("Received code is not valid."); - } - } - } - - // Save the transformed Cell for transform(): - this.transformedKV = transformed; + /** + * Internal implementation of {@link #filterKeyValue(Cell)}. Compared to the + * {@link #filterKeyValue(Cell)} method, this method accepts an additional parameter named + * currentTransformedCell. This parameter indicates the initial value of transformed cell before + * this filter operation.
+ * For FilterList, we can consider a filter list as a node in a tree. sub-filters of the filter + * list are children of the relative node. The logic of transforming cell of a filter list, well, + * we can consider it as the process of post-order tree traverse. For a node , Before we traverse + * the current child, we should set the traverse result (transformed cell) of previous node(s) as + * the initial value. so the additional currentTransformedCell parameter is needed (HBASE-18879). + * @param c The cell in question. + * @param transformedCell The transformed cell of previous filter(s) + * @return ReturnCode of this filter operation. + * @throws IOException + */ + ReturnCode internalFilterKeyValue(Cell c, Cell transformedCell) throws IOException { + return this.filterListBase.internalFilterKeyValue(c, transformedCell); + } - /* - * The seenNonHintReturnCode flag is intended only for Operator.MUST_PASS_ONE branch. - * If we have seen non SEEK_NEXT_USING_HINT ReturnCode, respect that ReturnCode. - */ - if (operator == Operator.MUST_PASS_ONE && !seenNonHintReturnCode) { - return ReturnCode.SEEK_NEXT_USING_HINT; - } - return rc; + @Override + public ReturnCode filterKeyValue(Cell c) throws IOException { + return filterListBase.filterKeyValue(c); } /** - * Filters that never filter by modifying the returned List of Cells can - * inherit this implementation that does nothing. - * - * {@inheritDoc} + * Filters that never filter by modifying the returned List of Cells can inherit this + * implementation that does nothing. {@inheritDoc} */ @Override public void filterRowCells(List cells) throws IOException { - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - filters.get(i).filterRowCells(cells); - } + filterListBase.filterRowCells(cells); } @Override public boolean hasFilterRow() { - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - if (filters.get(i).hasFilterRow()) { - return true; - } - } - return false; + return filterListBase.hasFilterRow(); } @Override public boolean filterRow() throws IOException { - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - Filter filter = filters.get(i); - if (operator == Operator.MUST_PASS_ALL) { - if (filter.filterRow()) { - return true; - } - } else if (operator == Operator.MUST_PASS_ONE) { - if (!filter.filterRow()) { - return false; - } - } - } - return operator == Operator.MUST_PASS_ONE; + return filterListBase.filterRow(); } /** * @return The filter serialized using pb */ public byte[] toByteArray() throws IOException { - FilterProtos.FilterList.Builder builder = - FilterProtos.FilterList.newBuilder(); + FilterProtos.FilterList.Builder builder = FilterProtos.FilterList.newBuilder(); builder.setOperator(FilterProtos.FilterList.Operator.valueOf(operator.name())); - int listize = filters.size(); - for (int i = 0; i < listize; i++) { + ArrayList filters = filterListBase.getFilters(); + for (int i = 0, n = filters.size(); i < n; i++) { builder.addFilters(ProtobufUtil.toFilter(filters.get(i))); } return builder.build().toByteArray(); @@ -396,8 +237,7 @@ final public class FilterList extends Filter { * @throws DeserializationException * @see #toByteArray */ - public static FilterList parseFrom(final byte [] pbBytes) - throws DeserializationException { + public static FilterList parseFrom(final byte[] pbBytes) throws DeserializationException { FilterProtos.FilterList proto; try { proto = FilterProtos.FilterList.parseFrom(pbBytes); @@ -416,96 +256,53 @@ final public class FilterList extends Filter { } catch (IOException ioe) { throw new DeserializationException(ioe); } - return new FilterList(Operator.valueOf(proto.getOperator().name()),rowFilters); + return new FilterList(Operator.valueOf(proto.getOperator().name()), rowFilters); } /** * @param other - * @return true if and only if the fields of the filter that are serialized - * are equal to the corresponding fields in other. Used for testing. + * @return true if and only if the fields of the filter that are serialized are equal to the + * corresponding fields in other. Used for testing. */ boolean areSerializedFieldsEqual(Filter other) { if (other == this) return true; if (!(other instanceof FilterList)) return false; - FilterList o = (FilterList)other; - return this.getOperator().equals(o.getOperator()) && - ((this.getFilters() == o.getFilters()) - || this.getFilters().equals(o.getFilters())); + FilterList o = (FilterList) other; + return this.getOperator().equals(o.getOperator()) + && ((this.getFilters() == o.getFilters()) || this.getFilters().equals(o.getFilters())); } @Override @Deprecated public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException { - return KeyValueUtil.ensureKeyValue(getNextCellHint((Cell)currentKV)); + return KeyValueUtil.ensureKeyValue(getNextCellHint((Cell) currentKV)); } @Override - public Cell getNextCellHint(Cell currentKV) throws IOException { - Cell keyHint = null; - if (operator == Operator.MUST_PASS_ALL) { - keyHint = seekHintFilter.getNextCellHint(currentKV); - return keyHint; - } - - // If any condition can pass, we need to keep the min hint - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - if (filters.get(i).filterAllRemaining()) { - continue; - } - Cell curKeyHint = filters.get(i).getNextCellHint(currentKV); - if (curKeyHint == null) { - // If we ever don't have a hint and this is must-pass-one, then no hint - return null; - } - if (curKeyHint != null) { - // If this is the first hint we find, set it - if (keyHint == null) { - keyHint = curKeyHint; - continue; - } - if (KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) { - keyHint = curKeyHint; - } - } - } - return keyHint; + public Cell getNextCellHint(Cell currentCell) throws IOException { + return this.filterListBase.getNextCellHint(currentCell); } @Override public boolean isFamilyEssential(byte[] name) throws IOException { - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - if (filters.get(i).isFamilyEssential(name)) { - return true; - } - } - return false; + return this.filterListBase.isFamilyEssential(name); } @Override public void setReversed(boolean reversed) { - int listize = filters.size(); - for (int i = 0; i < listize; i++) { - filters.get(i).setReversed(reversed); - } this.reversed = reversed; + this.filterListBase.setReversed(reversed); } @Override - public String toString() { - return toString(MAX_LOG_FILTERS); + public boolean isReversed() { + assert this.reversed == this.filterListBase.isReversed(); + return this.reversed; } - protected String toString(int maxFilters) { - int endIndex = this.filters.size() < maxFilters - ? this.filters.size() : maxFilters; - return String.format("%s %s (%d/%d): %s", - this.getClass().getSimpleName(), - this.operator == Operator.MUST_PASS_ALL ? "AND" : "OR", - endIndex, - this.filters.size(), - this.filters.subList(0, endIndex).toString()); + @Override + public String toString() { + return this.filterListBase.toString(); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java new file mode 100644 index 0000000..7ff7be0 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java @@ -0,0 +1,172 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Base class for FilterList. Currently, we have two sub-classes to extend this class: + * {@link FilterListWithOR}, {@link FilterListWithAND}. + */ +@InterfaceAudience.Private +public abstract class FilterListBase extends FilterBase { + private static final int MAX_LOG_FILTERS = 5; + protected final ArrayList filters; + + /** Reference Cell used by {@link #transformCell(Cell)} for validation purpose. */ + protected Cell referenceCell = null; + + /** + * When filtering a given Cell in {@link #filterKeyValue(Cell)}, this stores the transformed Cell + * to be returned by {@link #transformCell(Cell)}. Individual filters transformation are applied + * only when the filter includes the Cell. Transformations are composed in the order specified by + * {@link #filters}. + */ + protected Cell transformedCell = null; + + public FilterListBase(List filters) { + reversed = checkAndGetReversed(filters, reversed); + this.filters = new ArrayList<>(filters); + } + + protected static boolean isInReturnCodes(ReturnCode testRC, ReturnCode... returnCodes) { + for (ReturnCode rc : returnCodes) { + if (testRC.equals(rc)) return true; + } + return false; + } + + protected static boolean checkAndGetReversed(List rowFilters, boolean defaultValue) { + if (rowFilters.isEmpty()) { + return defaultValue; + } + boolean retValue = rowFilters.get(0).isReversed(); + for (Filter filter : rowFilters) { + if (filter.isReversed() != retValue) { + throw new IllegalArgumentException("Filters in the list must have the same reversed flag"); + } + } + return retValue; + } + + public abstract void addFilterLists(List filters); + + public int size() { + return this.filters.size(); + } + + public boolean isEmpty() { + return this.filters.isEmpty(); + } + + public ArrayList getFilters() { + return this.filters; + } + + protected int compareCell(Cell a, Cell b) { + int cmp = CellComparator.compare(a, b, false); + return reversed ? -1 * cmp : cmp; + } + + @Override + public Cell transformCell(Cell c) throws IOException { + if (isEmpty()) { + return super.transformCell(c); + } + if (!CellUtil.equals(c, referenceCell)) { + throw new IllegalStateException( + "Reference Cell: " + this.referenceCell + " does not match: " + c); + } + // Copy transformedCell into a new cell and reset transformedCell & referenceCell to null for + // Java GC optimization + Cell cell = KeyValueUtil.copyToNewKeyValue(this.transformedCell); + this.transformedCell = null; + this.referenceCell = null; + return cell; + } + + /** + * Internal implementation of {@link #filterKeyValue(Cell)} + * @param c The cell in question. + * @param transformedCell The transformed cell of previous filter(s) + * @return ReturnCode of this filter operation. + * @throws IOException + * @see org.apache.hadoop.hbase.filter.FilterList#internalFilterKeyValue(Cell, Cell) + */ + abstract ReturnCode internalFilterKeyValue(Cell c, Cell transformedCell) throws IOException; + + /** + * Filters that never filter by modifying the returned List of Cells can inherit this + * implementation that does nothing. {@inheritDoc} + */ + @Override + public void filterRowCells(List cells) throws IOException { + for (int i = 0, n = filters.size(); i < n; i++) { + filters.get(i).filterRowCells(cells); + } + } + + @Override + public boolean hasFilterRow() { + for (int i = 0, n = filters.size(); i < n; i++) { + if (filters.get(i).hasFilterRow()) { + return true; + } + } + return false; + } + + @Override + public boolean isFamilyEssential(byte[] name) throws IOException { + if (this.filters.isEmpty()) { + return super.isFamilyEssential(name); + } + for (int i = 0, n = filters.size(); i < n; i++) { + if (filters.get(i).isFamilyEssential(name)) { + return true; + } + } + return false; + } + + @Override + public void setReversed(boolean reversed) { + for (int i = 0, n = filters.size(); i < n; i++) { + filters.get(i).setReversed(reversed); + } + this.reversed = reversed; + } + + @Override + public String toString() { + int endIndex = this.size() < MAX_LOG_FILTERS ? this.size() : MAX_LOG_FILTERS; + return formatLogFilters(filters.subList(0, endIndex)); + } + + protected abstract String formatLogFilters(List logFilters); +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java new file mode 100644 index 0000000..bc04368 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java @@ -0,0 +1,292 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * FilterListWithAND represents an ordered list of filters which will be evaluated with an AND + * operator. + */ +@InterfaceAudience.Private +public class FilterListWithAND extends FilterListBase { + + private Set seekHintFilter = new HashSet<>(); + + public FilterListWithAND(List filters) { + super(filters); + } + + @Override + public void addFilterLists(List filters) { + if (checkAndGetReversed(filters, isReversed()) != isReversed()) { + throw new IllegalArgumentException("Filters in the list must have the same reversed flag"); + } + this.filters.addAll(filters); + } + + @Override + protected String formatLogFilters(List logFilters) { + return String.format("FilterList AND (%d/%d): %s", logFilters.size(), this.size(), + logFilters.toString()); + } + + /** + * FilterList with MUST_PASS_ALL choose the maximal forward step among sub-filters in filter list. + * Let's call it: The Maximal Step Rule. So if filter-A in filter list return INCLUDE and filter-B + * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return + * INCLUDE_AND_NEXT_COL. For SEEK_NEXT_USING_HINT, it's more special, and in method + * filterKeyValueWithMustPassAll(), if any sub-filter return SEEK_NEXT_USING_HINT, then our filter + * list will return SEEK_NEXT_USING_HINT. so we don't care about the SEEK_NEXT_USING_HINT here. + *
+ *
+ * The jump step will be: + * + *
+   * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
+   * 
+ * + * Here, we have the following map to describe The Maximal Step Rule. if current return code (for + * previous sub-filters in filter list) is ReturnCode, and current filter returns + * localRC, then we should return map[ReturnCode][localRC] for the merged result, + * according to The Maximal Step Rule.
+ * + *
+   * LocalCode\ReturnCode       INCLUDE                    INCLUDE_AND_NEXT_COL      INCLUDE_AND_SEEK_NEXT_ROW  SKIP                  NEXT_COL              NEXT_ROW              SEEK_NEXT_USING_HINT
+   * INCLUDE                    INCLUDE                    INCLUDE_AND_NEXT_COL      INCLUDE_AND_SEEK_NEXT_ROW  SKIP                  NEXT_COL              NEXT_ROW              SEEK_NEXT_USING_HINT
+   * INCLUDE_AND_NEXT_COL       INCLUDE_AND_NEXT_COL       INCLUDE_AND_NEXT_COL      INCLUDE_AND_SEEK_NEXT_ROW  NEXT_COL              NEXT_COL              NEXT_ROW              SEEK_NEXT_USING_HINT
+   * INCLUDE_AND_SEEK_NEXT_ROW  INCLUDE_AND_SEEK_NEXT_ROW  INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW  NEXT_ROW              NEXT_ROW              NEXT_ROW              SEEK_NEXT_USING_HINT
+   * SKIP                       SKIP                       NEXT_COL                  NEXT_ROW                   SKIP                  NEXT_COL              NEXT_ROW              SEEK_NEXT_USING_HINT
+   * NEXT_COL                   NEXT_COL                   NEXT_COL                  NEXT_ROW                   NEXT_COL              NEXT_COL              NEXT_ROW              SEEK_NEXT_USING_HINT
+   * NEXT_ROW                   NEXT_ROW                   NEXT_ROW                  NEXT_ROW                   NEXT_ROW              NEXT_ROW              NEXT_ROW              SEEK_NEXT_USING_HINT
+   * SEEK_NEXT_USING_HINT       SEEK_NEXT_USING_HINT       SEEK_NEXT_USING_HINT      SEEK_NEXT_USING_HINT       SEEK_NEXT_USING_HINT  SEEK_NEXT_USING_HINT  SEEK_NEXT_USING_HINT  SEEK_NEXT_USING_HINT
+   * 
+ * + * @param rc Return code which is calculated by previous sub-filter(s) in filter list. + * @param localRC Return code of the current sub-filter in filter list. + * @return Return code which is merged by the return code of previous sub-filter(s) and the return + * code of current sub-filter. + */ + private ReturnCode mergeReturnCode(ReturnCode rc, ReturnCode localRC) { + if (rc == ReturnCode.SEEK_NEXT_USING_HINT || localRC == ReturnCode.SEEK_NEXT_USING_HINT) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + switch (localRC) { + case INCLUDE: + return rc; + case INCLUDE_AND_NEXT_COL: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL)) { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW; + } + if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL)) { + return ReturnCode.NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) { + return ReturnCode.NEXT_ROW; + } + break; + case INCLUDE_AND_SEEK_NEXT_ROW: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW; + } + if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) { + return ReturnCode.NEXT_ROW; + } + break; + case SKIP: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP)) { + return ReturnCode.SKIP; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) { + return ReturnCode.NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) { + return ReturnCode.NEXT_ROW; + } + break; + case NEXT_COL: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP, + ReturnCode.NEXT_COL)) { + return ReturnCode.NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) { + return ReturnCode.NEXT_ROW; + } + break; + case NEXT_ROW: + return ReturnCode.NEXT_ROW; + } + throw new IllegalStateException( + "Received code is not valid. rc: " + rc + ", localRC: " + localRC); + } + + private ReturnCode filterKeyValueWithMustPassAll(Cell c) throws IOException { + ReturnCode rc = ReturnCode.INCLUDE; + Cell transformed = c; + this.seekHintFilter.clear(); + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + if (filter.filterAllRemaining()) { + return ReturnCode.NEXT_ROW; + } + ReturnCode localRC = filter.filterKeyValue(c); + rc = mergeReturnCode(rc, localRC); + + // For INCLUDE* case, we need to update the transformed cell. + if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + transformed = filter.transformCell(transformed); + } + if (localRC == ReturnCode.SEEK_NEXT_USING_HINT) { + seekHintFilter.add(filter); + } + } + this.transformedCell = transformed; + if (!seekHintFilter.isEmpty()) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + return rc; + } + + @Override + ReturnCode internalFilterKeyValue(Cell c, Cell transformedCell) throws IOException { + if (isEmpty()) { + return ReturnCode.INCLUDE; + } + ReturnCode rc = ReturnCode.INCLUDE; + Cell transformed = transformedCell; + this.referenceCell = c; + this.seekHintFilter.clear(); + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + if (filter.filterAllRemaining()) { + return ReturnCode.NEXT_ROW; + } + ReturnCode localRC; + if (filter instanceof FilterList) { + localRC = ((FilterList) filter).internalFilterKeyValue(c, transformed); + } else { + localRC = filter.filterKeyValue(c); + } + rc = mergeReturnCode(rc, localRC); + + // For INCLUDE* case, we need to update the transformed cell. + if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + transformed = filter.transformCell(transformed); + } + if (localRC == ReturnCode.SEEK_NEXT_USING_HINT) { + seekHintFilter.add(filter); + } + } + this.transformedCell = transformed; + if (!seekHintFilter.isEmpty()) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + return rc; + } + + @Override + public ReturnCode filterKeyValue(Cell c) throws IOException { + return internalFilterKeyValue(c, c); + } + + @Override + public void reset() throws IOException { + for (int i = 0, n = filters.size(); i < n; i++) { + filters.get(i).reset(); + } + seekHintFilter.clear(); + } + + @Override + public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException { + if (isEmpty()) { + return super.filterRowKey(rowKey, offset, length); + } + boolean retVal = false; + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + if (filter.filterAllRemaining() || filter.filterRowKey(rowKey, offset, length)) { + retVal = true; + } + } + return retVal; + } + + @Override + public boolean filterAllRemaining() throws IOException { + if (isEmpty()) { + return super.filterAllRemaining(); + } + for (int i = 0, n = filters.size(); i < n; i++) { + if (filters.get(i).filterAllRemaining()) { + return true; + } + } + return false; + } + + @Override + public boolean filterRow() throws IOException { + if (isEmpty()) { + return super.filterRow(); + } + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + if (filter.filterRow()) { + return true; + } + } + return false; + } + + @Override + public Cell getNextCellHint(Cell currentCell) throws IOException { + if (isEmpty()) { + return super.getNextCellHint(currentCell); + } + Cell maxHint = null; + for (Filter filter : seekHintFilter) { + if (filter.filterAllRemaining()) { + continue; + } + Cell curKeyHint = filter.getNextCellHint(currentCell); + if (maxHint == null) { + maxHint = curKeyHint; + continue; + } + if (this.compareCell(maxHint, curKeyHint) < 0) { + maxHint = curKeyHint; + } + } + return maxHint; + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java new file mode 100644 index 0000000..4b85aa8 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java @@ -0,0 +1,424 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * FilterListWithOR represents an ordered list of filters which will be evaluated with an OR + * operator. + */ +@InterfaceAudience.Private +public class FilterListWithOR extends FilterListBase { + + /** + * Save previous return code and previous cell for every filter in filter list. For MUST_PASS_ONE, + * we use the previous return code to decide whether we should pass current cell encountered to + * the filter. For MUST_PASS_ALL, the two list are meaningless. + */ + private List prevFilterRCList = null; + private List prevCellList = null; + + public FilterListWithOR(List filters) { + super(filters); + prevFilterRCList = + new ArrayList(Collections.nCopies(filters.size(), (ReturnCode) null)); + prevCellList = new ArrayList(Collections.nCopies(filters.size(), (Cell) null)); + } + + @Override + public void addFilterLists(List filters) { + if (checkAndGetReversed(filters, isReversed()) != isReversed()) { + throw new IllegalArgumentException("Filters in the list must have the same reversed flag"); + } + this.filters.addAll(filters); + this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), (ReturnCode) null)); + this.prevCellList.addAll(Collections.nCopies(filters.size(), (Cell) null)); + } + + @Override + protected String formatLogFilters(List logFilters) { + return String.format("FilterList OR (%d/%d): %s", logFilters.size(), this.size(), + logFilters.toString()); + } + + /** + * For MUST_PASS_ONE, we cannot make sure that when filter-A in filter list return NEXT_COL then + * the next cell passing to filterList will be the first cell in next column, because if filter-B + * in filter list return SKIP, then the filter list will return SKIP. In this case, we should pass + * the cell following the previous cell, and it's possible that the next cell has the same column + * as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should + * save the previous cell and the return code list when checking previous cell for every filter in + * filter list, and verify if currentCell fit the previous return code, if fit then pass the + * currentCell to the corresponding filter. (HBASE-17678)
+ * Note that: In StoreScanner level, NEXT_ROW will skip to the next row in current family, and in + * RegionScanner level, NEXT_ROW will skip to the next row in current family and switch to the + * next family for RegionScanner, INCLUDE_AND_NEXT_ROW is the same. so we should pass current cell + * to the filter, if row mismatch or row match but column family mismatch. (HBASE-18368) + * @see org.apache.hadoop.hbase.filter.Filter.ReturnCode + */ + private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx) + throws IOException { + ReturnCode prevCode = this.prevFilterRCList.get(filterIdx); + if (prevCell == null || prevCode == null) { + return true; + } + switch (prevCode) { + case INCLUDE: + case SKIP: + return true; + case SEEK_NEXT_USING_HINT: + Cell nextHintCell = getNextCellHint(prevCell); + return nextHintCell == null || this.compareCell(currentCell, nextHintCell) >= 0; + case NEXT_COL: + case INCLUDE_AND_NEXT_COL: + return !CellUtil.matchingRowColumn(prevCell, currentCell); + case NEXT_ROW: + case INCLUDE_AND_SEEK_NEXT_ROW: + return !CellUtil.matchingRow(prevCell, currentCell) + || !CellUtil.matchingFamily(prevCell, currentCell); + default: + throw new IllegalStateException("Received code is not valid."); + } + } + + /** + * FilterList with MUST_PASS_ONE choose the minimal forward step among sub-filter in filter list. + * Let's call it: The Minimal Step Rule. So if filter-A in filter list return INCLUDE and filter-B + * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return INCLUDE. For + * SEEK_NEXT_USING_HINT, it's more special, because we do not know how far it will forward, so we + * use SKIP by default.
+ *
+ * The jump step will be: + * + *
+   * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
+   * 
+ * + * Here, we have the following map to describe The Minimal Step Rule. if current return code (for + * previous sub-filters in filter list) is ReturnCode, and current filter returns + * localRC, then we should return map[ReturnCode][localRC] for the merged result, + * according to The Minimal Step Rule.
+ * + *
+   * LocalCode\ReturnCode       INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_SEEK_NEXT_ROW  SKIP      NEXT_COL              NEXT_ROW                  SEEK_NEXT_USING_HINT
+   * INCLUDE                    INCLUDE INCLUDE                  INCLUDE                    INCLUDE   INCLUDE               INCLUDE                   INCLUDE
+   * INCLUDE_AND_NEXT_COL       INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_NEXT_COL       INCLUDE   INCLUDE_AND_NEXT_COL  INCLUDE_AND_NEXT_COL      INCLUDE
+   * INCLUDE_AND_SEEK_NEXT_ROW  INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_SEEK_NEXT_ROW  INCLUDE   INCLUDE_AND_NEXT_COL  INCLUDE_AND_SEEK_NEXT_ROW INCLUDE
+   * SKIP                       INCLUDE INCLUDE                  INCLUDE                    SKIP      SKIP                  SKIP                      SKIP
+   * NEXT_COL                   INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_NEXT_COL       SKIP      NEXT_COL              NEXT_COL                  SKIP
+   * NEXT_ROW                   INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_SEEK_NEXT_ROW  SKIP      NEXT_COL              NEXT_ROW                  SKIP
+   * SEEK_NEXT_USING_HINT       INCLUDE INCLUDE                  INCLUDE                    SKIP      SKIP                  SKIP                      SEEK_NEXT_USING_HINT
+   * 
+ * + * @param rc Return code which is calculated by previous sub-filter(s) in filter list. + * @param localRC Return code of the current sub-filter in filter list. + * @return Return code which is merged by the return code of previous sub-filter(s) and the return + * code of current sub-filter. + */ + private ReturnCode mergeReturnCode(ReturnCode rc, ReturnCode localRC) { + if (rc == null) return localRC; + switch (localRC) { + case INCLUDE: + return ReturnCode.INCLUDE; + case INCLUDE_AND_NEXT_COL: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP, + ReturnCode.SEEK_NEXT_USING_HINT)) { + return ReturnCode.INCLUDE; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, + ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + break; + case INCLUDE_AND_SEEK_NEXT_ROW: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP, + ReturnCode.SEEK_NEXT_USING_HINT)) { + return ReturnCode.INCLUDE; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) { + return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW; + } + break; + case SKIP: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + return ReturnCode.INCLUDE; + } + if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, + ReturnCode.SEEK_NEXT_USING_HINT)) { + return ReturnCode.SKIP; + } + break; + case NEXT_COL: + if (isInReturnCodes(rc, ReturnCode.INCLUDE)) { + return ReturnCode.INCLUDE; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) { + return ReturnCode.SKIP; + } + break; + case NEXT_ROW: + if (isInReturnCodes(rc, ReturnCode.INCLUDE)) { + return ReturnCode.INCLUDE; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL)) { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW; + } + if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) { + return ReturnCode.SKIP; + } + if (isInReturnCodes(rc, ReturnCode.NEXT_COL)) { + return ReturnCode.NEXT_COL; + } + if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) { + return ReturnCode.NEXT_ROW; + } + break; + case SEEK_NEXT_USING_HINT: + if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + return ReturnCode.INCLUDE; + } + if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) { + return ReturnCode.SKIP; + } + if (isInReturnCodes(rc, ReturnCode.SEEK_NEXT_USING_HINT)) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + break; + } + throw new IllegalStateException( + "Received code is not valid. rc: " + rc + ", localRC: " + localRC); + } + + private void updatePrevFilterRCList(int index, ReturnCode currentRC) { + prevFilterRCList.set(index, currentRC); + } + + private void updatePrevCellList(int index, Cell currentCell, ReturnCode currentRC) { + if (currentCell == null || currentRC == ReturnCode.INCLUDE || currentRC == ReturnCode.SKIP) { + // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the + // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we + // need not save current cell to prevCellList for saving heap memory. + prevCellList.set(index, null); + } else { + prevCellList.set(index, KeyValueUtil.copyToNewKeyValue(currentCell)); + } + } + + private ReturnCode filterKeyValueWithMustPassOne(Cell c) throws IOException { + ReturnCode rc = null; + boolean everyFilterReturnHint = true; + Cell transformed = c; + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + + Cell prevCell = this.prevCellList.get(i); + if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) { + everyFilterReturnHint = false; + continue; + } + + ReturnCode localRC = filter.filterKeyValue(c); + + // Update previous return code and previous cell for filter[i]. + updatePrevFilterRCList(i, localRC); + updatePrevCellList(i, c, localRC); + + if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) { + everyFilterReturnHint = false; + } + + rc = mergeReturnCode(rc, localRC); + + // For INCLUDE* case, we need to update the transformed cell. + if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + transformed = filter.transformCell(transformed); + } + } + + this.transformedCell = transformed; + if (everyFilterReturnHint) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } else if (rc == null) { + // Each sub-filter in filter list got true for filterAllRemaining(). + return ReturnCode.SKIP; + } else { + return rc; + } + } + + @Override + ReturnCode internalFilterKeyValue(Cell c, Cell transformedCell) throws IOException { + if (isEmpty()) { + return ReturnCode.INCLUDE; + } + ReturnCode rc = null; + boolean everyFilterReturnHint = true; + Cell transformed = transformedCell; + this.referenceCell = c; + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + + Cell prevCell = this.prevCellList.get(i); + if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) { + everyFilterReturnHint = false; + continue; + } + + ReturnCode localRC; + if (filter instanceof FilterList) { + localRC = ((FilterList) filter).internalFilterKeyValue(c, transformed); + } else { + localRC = filter.filterKeyValue(c); + } + + // Update previous return code and previous cell for filter[i]. + updatePrevFilterRCList(i, localRC); + updatePrevCellList(i, c, localRC); + + if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) { + everyFilterReturnHint = false; + } + + rc = mergeReturnCode(rc, localRC); + + // For INCLUDE* case, we need to update the transformed cell. + if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, + ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) { + transformed = filter.transformCell(transformed); + } + } + + this.transformedCell = transformed; + if (everyFilterReturnHint) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } else if (rc == null) { + // Each sub-filter in filter list got true for filterAllRemaining(). + return ReturnCode.SKIP; + } else { + return rc; + } + } + + @Override + public ReturnCode filterKeyValue(Cell c) throws IOException { + return internalFilterKeyValue(c, c); + } + + @Override + public void reset() throws IOException { + for (int i = 0, n = filters.size(); i < n; i++) { + filters.get(i).reset(); + prevFilterRCList.set(i, null); + prevCellList.set(i, null); + } + } + + @Override + public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException { + if (isEmpty()) { + return super.filterRowKey(rowKey, offset, length); + } + boolean retVal = true; + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey, offset, length)) { + retVal = false; + } + } + return retVal; + } + + @Override + public boolean filterAllRemaining() throws IOException { + if (isEmpty()) { + return super.filterAllRemaining(); + } + for (int i = 0, n = filters.size(); i < n; i++) { + if (!filters.get(i).filterAllRemaining()) { + return false; + } + } + return true; + } + + @Override + public boolean filterRow() throws IOException { + if (isEmpty()) { + return super.filterRow(); + } + for (int i = 0, n = filters.size(); i < n; i++) { + Filter filter = filters.get(i); + if (!filter.filterRow()) { + return false; + } + } + return true; + } + + @Override + public Cell getNextCellHint(Cell currentCell) throws IOException { + if (isEmpty()) { + return super.getNextCellHint(currentCell); + } + Cell minKeyHint = null; + // If any condition can pass, we need to keep the min hint + for (int i = 0, n = filters.size(); i < n; i++) { + if (filters.get(i).filterAllRemaining()) { + continue; + } + Cell curKeyHint = filters.get(i).getNextCellHint(currentCell); + if (curKeyHint == null) { + // If we ever don't have a hint and this is must-pass-one, then no hint + return null; + } + // If this is the first hint we find, set it + if (minKeyHint == null) { + minKeyHint = curKeyHint; + continue; + } + if (this.compareCell(minKeyHint, curKeyHint) > 0) { + minKeyHint = curKeyHint; + } + } + return minKeyHint; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java index 6ddc422..960394f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java @@ -25,7 +25,6 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNull; @@ -39,6 +38,7 @@ import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -46,16 +46,10 @@ import com.google.common.collect.Lists; /** * Tests filter sets - * */ @Category(SmallTests.class) public class TestFilterList { static final int MAX_PAGES = 2; - static final char FIRST_CHAR = 'a'; - static final char LAST_CHAR = 'e'; - static byte[] GOOD_BYTES = Bytes.toBytes("abc"); - static byte[] BAD_BYTES = Bytes.toBytes("def"); - @Test public void testAddFilter() throws Exception { @@ -73,10 +67,8 @@ public class TestFilterList { filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2)); filterList.addFilter(new FirstKeyOnlyFilter()); - } - /** * Test "must pass one" * @throws Exception @@ -90,33 +82,30 @@ public class TestFilterList { List filters = new ArrayList(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); - Filter filterMPONE = - new FilterList(FilterList.Operator.MUST_PASS_ONE, filters); - return filterMPONE; + return new FilterList(FilterList.Operator.MUST_PASS_ONE, filters); } + /** + * Filter must do all below steps: + *
    + *
  • Filter.reset()
  • + *
  • Filter.filterAllRemaining() -> true indicates scan is over, false, keep going on.
  • + *
  • Filter.filterRowKey(byte[],int,int) -> true to drop this row, if false, we will also + * call
  • + *
  • Filter.filterKeyValue(org.apache.hadoop.hbase.KeyValue) -> true to drop this key/value
  • + *
  • Filter.filterRow() -> last chance to drop entire row based on the sequence of filterValue() + * calls. Eg: filter a row if it doesn't contain a specified column.
  • + *
+ */ private void mpOneTest(Filter filterMPONE) throws Exception { - /* Filter must do all below steps: - *
    - *
  • {@link #reset()}
  • - *
  • {@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.
  • - *
  • {@link #filterRowKey(byte[],int,int)} -> true to drop this row, - * if false, we will also call
  • - *
  • {@link #filterKeyValue(org.apache.hadoop.hbase.KeyValue)} -> true to drop this key/value
  • - *
  • {@link #filterRow()} -> last chance to drop entire row based on the sequence of - * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. - *
  • - *
- */ filterMPONE.reset(); assertFalse(filterMPONE.filterAllRemaining()); /* Will pass both */ - byte [] rowkey = Bytes.toBytes("yyyyyyyyy"); + byte[] rowkey = Bytes.toBytes("yyyyyyyyy"); for (int i = 0; i < MAX_PAGES - 1; i++) { assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); - KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), - Bytes.toBytes(i)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); assertFalse(filterMPONE.filterRow()); } @@ -124,16 +113,14 @@ public class TestFilterList { /* Only pass PageFilter */ rowkey = Bytes.toBytes("z"); assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); - KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), - Bytes.toBytes(0)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), Bytes.toBytes(0)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); assertFalse(filterMPONE.filterRow()); /* reach MAX_PAGES already, should filter any rows */ rowkey = Bytes.toBytes("yyy"); assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); - kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), - Bytes.toBytes(0)); + kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), Bytes.toBytes(0)); assertFalse(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); assertFalse(filterMPONE.filterRow()); @@ -156,31 +143,30 @@ public class TestFilterList { List filters = new ArrayList(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); - Filter filterMPALL = - new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); + Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); return filterMPALL; } + /** + * Filter must do all below steps: + *
    + *
  • Filter.reset()
  • + *
  • Filter.filterAllRemaining() -> true indicates scan is over, false, keep going on.
  • + *
  • Filter.filterRowKey(byte[],int,int) -> true to drop this row, if false, we will also + * call
  • + *
  • Filter.filterKeyValue(org.apache.hadoop.hbase.KeyValue) -> true to drop this key/value
  • + *
  • Filter.filterRow() -> last chance to drop entire row based on the sequence of filterValue() + * calls. Eg: filter a row if it doesn't contain a specified column.
  • + *
+ */ private void mpAllTest(Filter filterMPALL) throws Exception { - /* Filter must do all below steps: - *
    - *
  • {@link #reset()}
  • - *
  • {@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.
  • - *
  • {@link #filterRowKey(byte[],int,int)} -> true to drop this row, - * if false, we will also call
  • - *
  • {@link #filterKeyValue(org.apache.hadoop.hbase.KeyValue)} -> true to drop this key/value
  • - *
  • {@link #filterRow()} -> last chance to drop entire row based on the sequence of - * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. - *
  • - *
- */ + filterMPALL.reset(); assertFalse(filterMPALL.filterAllRemaining()); - byte [] rowkey = Bytes.toBytes("yyyyyyyyy"); + byte[] rowkey = Bytes.toBytes("yyyyyyyyy"); for (int i = 0; i < MAX_PAGES - 1; i++) { assertFalse(filterMPALL.filterRowKey(rowkey, 0, rowkey.length)); - KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), - Bytes.toBytes(i)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterKeyValue(kv)); } filterMPALL.reset(); @@ -204,34 +190,32 @@ public class TestFilterList { List filters = new ArrayList(); filters.add(new PrefixFilter(Bytes.toBytes("yyy"))); filters.add(new PageFilter(MAX_PAGES)); - Filter filterMPONE = - new FilterList(FilterList.Operator.MUST_PASS_ONE, filters); + Filter filterMPONE = new FilterList(FilterList.Operator.MUST_PASS_ONE, filters); return filterMPONE; } + /** + * Filter must do all below steps: + *
    + *
  • Filter.reset()
  • + *
  • Filter.filterAllRemaining() -> true indicates scan is over, false, keep going on.
  • + *
  • Filter.filterRowKey(byte[],int,int) -> true to drop this row, if false, we will also + * call
  • + *
  • Filter.filterKeyValue(org.apache.hadoop.hbase.KeyValue) -> true to drop this key/value
  • + *
  • Filter.filterRow() -> last chance to drop entire row based on the sequence of filterValue() + * calls. Eg: filter a row if it doesn't contain a specified column.
  • + *
+ */ public void orderingTest(Filter filterMPONE) throws Exception { - /* Filter must do all below steps: - *
    - *
  • {@link #reset()}
  • - *
  • {@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.
  • - *
  • {@link #filterRowKey(byte[],int,int)} -> true to drop this row, - * if false, we will also call
  • - *
  • {@link #filterKeyValue(org.apache.hadoop.hbase.KeyValue)} -> true to drop this key/value
  • - *
  • {@link #filterRow()} -> last chance to drop entire row based on the sequence of - * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. - *
  • - *
- */ filterMPONE.reset(); assertFalse(filterMPONE.filterAllRemaining()); /* We should be able to fill MAX_PAGES without incrementing page counter */ - byte [] rowkey = Bytes.toBytes("yyyyyyyy"); + byte[] rowkey = Bytes.toBytes("yyyyyyyy"); for (int i = 0; i < MAX_PAGES; i++) { assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); - KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), - Bytes.toBytes(i)); - assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); assertFalse(filterMPONE.filterRow()); } @@ -239,9 +223,8 @@ public class TestFilterList { rowkey = Bytes.toBytes("xxxxxxx"); for (int i = 0; i < MAX_PAGES; i++) { assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); - KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), - Bytes.toBytes(i)); - assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); assertFalse(filterMPONE.filterRow()); } @@ -249,16 +232,15 @@ public class TestFilterList { rowkey = Bytes.toBytes("yyy"); for (int i = 0; i < MAX_PAGES; i++) { assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); - KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), - Bytes.toBytes(i)); - assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); assertFalse(filterMPONE.filterRow()); } } /** - * When we do a "MUST_PASS_ONE" (a logical 'OR') of the above two filters - * we expect to get the same result as the 'prefix' only result. + * When we do a "MUST_PASS_ONE" (a logical 'OR') of the above two filters we expect to get the + * same result as the 'prefix' only result. * @throws Exception */ @Test @@ -266,61 +248,63 @@ public class TestFilterList { byte[] r1 = Bytes.toBytes("Row1"); byte[] r11 = Bytes.toBytes("Row11"); byte[] r2 = Bytes.toBytes("Row2"); - + FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new PrefixFilter(r1)); flist.filterRowKey(r1, 0, r1.length); - assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE); - assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE); + assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1))); + assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11))); flist.reset(); flist.filterRowKey(r2, 0, r2.length); - assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP); - + assertEquals(ReturnCode.SKIP, flist.filterKeyValue(new KeyValue(r2, r2, r2))); + flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new AlwaysNextColFilter()); flist.addFilter(new PrefixFilter(r1)); flist.filterRowKey(r1, 0, r1.length); - assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE); - assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE); + assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1))); + assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11))); flist.reset(); flist.filterRowKey(r2, 0, r2.length); - assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP); + assertEquals(ReturnCode.NEXT_COL, flist.filterKeyValue(new KeyValue(r2, r2, r2))); } /** - * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters - * we expect to get the same result as the inclusive stop result. + * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters we expect to get the same + * result as the inclusive stop result. * @throws Exception */ @Test - public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception { + public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception { byte[] r1 = Bytes.toBytes("Row1"); byte[] r11 = Bytes.toBytes("Row11"); byte[] r2 = Bytes.toBytes("Row2"); - + FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new AlwaysNextColFilter()); flist.addFilter(new InclusiveStopFilter(r1)); flist.filterRowKey(r1, 0, r1.length); - assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE); - assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE); + assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1))); + assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11))); flist.reset(); flist.filterRowKey(r2, 0, r2.length); - assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP); + assertEquals(ReturnCode.NEXT_COL, flist.filterKeyValue(new KeyValue(r2, r2, r2))); } - public static class AlwaysNextColFilter extends FilterBase { + static class AlwaysNextColFilter extends FilterBase { public AlwaysNextColFilter() { super(); } + @Override public ReturnCode filterKeyValue(Cell v) { return ReturnCode.NEXT_COL; } - public static AlwaysNextColFilter parseFrom(final byte [] pbBytes) + + public static AlwaysNextColFilter parseFrom(final byte[] pbBytes) throws DeserializationException { return new AlwaysNextColFilter(); } @@ -335,14 +319,13 @@ public class TestFilterList { List filters = new ArrayList(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); - Filter filterMPALL = - new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); + Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); // Decompose filterMPALL to bytes. byte[] buffer = filterMPALL.toByteArray(); // Recompose filterMPALL. - FilterList newFilter = FilterList.parseFrom(buffer); + FilterList.parseFrom(buffer); // Run tests mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE()))); @@ -368,8 +351,8 @@ public class TestFilterList { @Override public Filter.ReturnCode filterKeyValue(Cell v) { - Filter.ReturnCode returnCode = returnInclude ? Filter.ReturnCode.INCLUDE : - Filter.ReturnCode.SKIP; + Filter.ReturnCode returnCode = + returnInclude ? Filter.ReturnCode.INCLUDE : Filter.ReturnCode.SKIP; returnInclude = !returnInclude; return returnCode; } @@ -380,8 +363,8 @@ public class TestFilterList { @Override public Filter.ReturnCode filterKeyValue(Cell v) { - Filter.ReturnCode returnCode = returnIncludeOnly ? Filter.ReturnCode.INCLUDE : - Filter.ReturnCode.INCLUDE_AND_NEXT_COL; + Filter.ReturnCode returnCode = + returnIncludeOnly ? Filter.ReturnCode.INCLUDE : Filter.ReturnCode.INCLUDE_AND_NEXT_COL; returnIncludeOnly = !returnIncludeOnly; return returnCode; } @@ -391,8 +374,8 @@ public class TestFilterList { FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE, Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter })); // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL. - assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null)); - // INCLUDE, SKIP, INCLUDE. + assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null)); + // INCLUDE, SKIP, INCLUDE. assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null)); // Check must pass all filter. @@ -400,7 +383,7 @@ public class TestFilterList { Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter })); // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL. assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null)); - // INCLUDE, SKIP, INCLUDE. + // INCLUDE, SKIP, INCLUDE. assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null)); } @@ -411,15 +394,14 @@ public class TestFilterList { public void testHintPassThru() throws Exception { final KeyValue minKeyValue = new KeyValue(Bytes.toBytes(0L), null, null); - final KeyValue maxKeyValue = new KeyValue(Bytes.toBytes(Long.MAX_VALUE), - null, null); + final KeyValue maxKeyValue = new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null); Filter filterNoHint = new FilterBase() { @Override - public byte [] toByteArray() { + public byte[] toByteArray() { return null; } - + @Override public ReturnCode filterKeyValue(Cell ignored) throws IOException { return ReturnCode.INCLUDE; @@ -438,7 +420,9 @@ public class TestFilterList { } @Override - public byte [] toByteArray() {return null;} + public byte[] toByteArray() { + return null; + } }; Filter filterMaxHint = new FilterBase() { @@ -453,92 +437,89 @@ public class TestFilterList { } @Override - public byte [] toByteArray() {return null;} + public byte[] toByteArray() { + return null; + } }; // MUST PASS ONE // Should take the min if given two hints FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, - Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } )); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - minKeyValue)); + Arrays.asList(new Filter[] { filterMinHint, filterMaxHint })); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), minKeyValue)); // Should have no hint if any filter has no hint filterList = new FilterList(Operator.MUST_PASS_ONE, - Arrays.asList( - new Filter [] { filterMinHint, filterMaxHint, filterNoHint } )); + Arrays.asList(new Filter[] { filterMinHint, filterMaxHint, filterNoHint })); assertNull(filterList.getNextKeyHint(null)); filterList = new FilterList(Operator.MUST_PASS_ONE, - Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } )); + Arrays.asList(new Filter[] { filterNoHint, filterMaxHint })); assertNull(filterList.getNextKeyHint(null)); // Should give max hint if its the only one filterList = new FilterList(Operator.MUST_PASS_ONE, - Arrays.asList(new Filter [] { filterMaxHint, filterMaxHint } )); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - maxKeyValue)); + Arrays.asList(new Filter[] { filterMaxHint, filterMaxHint })); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); // MUST PASS ALL // Should take the first hint filterList = new FilterList(Operator.MUST_PASS_ALL, - Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } )); + Arrays.asList(new Filter[] { filterMinHint, filterMaxHint })); filterList.filterKeyValue(null); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - minKeyValue)); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); filterList = new FilterList(Operator.MUST_PASS_ALL, - Arrays.asList(new Filter [] { filterMaxHint, filterMinHint } )); + Arrays.asList(new Filter[] { filterMaxHint, filterMinHint })); filterList.filterKeyValue(null); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - maxKeyValue)); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); // Should have first hint even if a filter has no hint filterList = new FilterList(Operator.MUST_PASS_ALL, - Arrays.asList( - new Filter [] { filterNoHint, filterMinHint, filterMaxHint } )); + Arrays.asList(new Filter[] { filterNoHint, filterMinHint, filterMaxHint })); filterList.filterKeyValue(null); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - minKeyValue)); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); filterList = new FilterList(Operator.MUST_PASS_ALL, - Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } )); + Arrays.asList(new Filter[] { filterNoHint, filterMaxHint })); filterList.filterKeyValue(null); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - maxKeyValue)); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), maxKeyValue)); filterList = new FilterList(Operator.MUST_PASS_ALL, - Arrays.asList(new Filter [] { filterNoHint, filterMinHint } )); + Arrays.asList(new Filter[] { filterNoHint, filterMinHint })); filterList.filterKeyValue(null); - assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), - minKeyValue)); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), minKeyValue)); } /** - * Tests the behavior of transform() in a hierarchical filter. - * - * transform() only applies after a filterKeyValue() whose return-code includes the KeyValue. - * Lazy evaluation of AND + * Tests the behavior of transform() in a hierarchical filter. transform() only applies after a + * filterKeyValue() whose return-code includes the KeyValue. Lazy evaluation of AND */ @Test public void testTransformMPO() throws Exception { // Apply the following filter: - // (family=fam AND qualifier=qual1 AND KeyOnlyFilter) - // OR (family=fam AND qualifier=qual2) - final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.newArrayList( - new FilterList(Operator.MUST_PASS_ALL, Lists.newArrayList( - new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))), - new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))), - new KeyOnlyFilter())), - new FilterList(Operator.MUST_PASS_ALL, Lists.newArrayList( - new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))), - new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual2"))))))); - - final KeyValue kvQual1 = new KeyValue( - Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value")); - final KeyValue kvQual2 = new KeyValue( - Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value")); - final KeyValue kvQual3 = new KeyValue( - Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value")); + // (family=fam AND qualifier=qual1 AND KeyOnlyFilter) + // OR (family=fam AND qualifier=qual2) + final FilterList flist = + new FilterList(Operator.MUST_PASS_ONE, + Lists. newArrayList( + new FilterList(Operator.MUST_PASS_ALL, + Lists. newArrayList( + new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))), + new QualifierFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("qual1"))), + new KeyOnlyFilter())), + new FilterList(Operator.MUST_PASS_ALL, + Lists. newArrayList( + new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))), + new QualifierFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("qual2"))))))); + + final KeyValue kvQual1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), + Bytes.toBytes("qual1"), Bytes.toBytes("value")); + final KeyValue kvQual2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), + Bytes.toBytes("qual2"), Bytes.toBytes("value")); + final KeyValue kvQual3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), + Bytes.toBytes("qual3"), Bytes.toBytes("value")); // Value for fam:qual1 should be stripped: assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual1)); @@ -554,5 +535,332 @@ public class TestFilterList { assertEquals(Filter.ReturnCode.SKIP, flist.filterKeyValue(kvQual3)); } -} + static class MockSeekHintFilter extends FilterBase { + private Cell returnCell; + + public MockSeekHintFilter(Cell returnCell) { + this.returnCell = returnCell; + } + + @Override + public ReturnCode filterKeyValue(Cell v) throws IOException { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + + @Override + public Cell getNextCellHint(Cell currentCell) throws IOException { + return this.returnCell; + } + } + + @Test + public void testReversedFilterListWithMockSeekHintFilter() throws IOException { + KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, + Bytes.toBytes("value")); + KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, + Bytes.toBytes("value")); + KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, + Bytes.toBytes("value")); + Filter filter1 = new MockSeekHintFilter(kv1); + filter1.setReversed(true); + Filter filter2 = new MockSeekHintFilter(kv2); + filter2.setReversed(true); + Filter filter3 = new MockSeekHintFilter(kv3); + filter3.setReversed(true); + + FilterList filterList = new FilterList(Operator.MUST_PASS_ONE); + filterList.setReversed(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + filterList.addFilter(filter3); + + Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1)); + Assert.assertEquals(kv3, filterList.getNextCellHint(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ALL); + filterList.setReversed(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + filterList.addFilter(filter3); + + Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1)); + Assert.assertEquals(kv1, filterList.getNextCellHint(kv1)); + } + + @Test + public void testReversedFilterListWithOR() throws IOException { + byte[] r22 = Bytes.toBytes("Row22"); + byte[] r2 = Bytes.toBytes("Row2"); + byte[] r1 = Bytes.toBytes("Row1"); + + FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); + filterList.setReversed(true); + PrefixFilter prefixFilter = new PrefixFilter(r2); + prefixFilter.setReversed(true); + filterList.addFilter(prefixFilter); + filterList.filterRowKey(r22, 0, r22.length); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r22, r22, r22))); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r2, r2, r2))); + + filterList.reset(); + filterList.filterRowKey(r1, 0, r1.length); + assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(new KeyValue(r1, r1, r1))); + + filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); + filterList.setReversed(true); + AlwaysNextColFilter alwaysNextColFilter = new AlwaysNextColFilter(); + alwaysNextColFilter.setReversed(true); + prefixFilter = new PrefixFilter(r2); + prefixFilter.setReversed(true); + filterList.addFilter(alwaysNextColFilter); + filterList.addFilter(prefixFilter); + filterList.filterRowKey(r22, 0, r22.length); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r22, r22, r22))); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r2, r2, r2))); + + filterList.reset(); + filterList.filterRowKey(r1, 0, r1.length); + assertEquals(ReturnCode.NEXT_COL, filterList.filterKeyValue(new KeyValue(r1, r1, r1))); + } + + private static class MockFilter extends FilterBase { + private ReturnCode targetRetCode; + public boolean didCellPassToTheFilter = false; + + public MockFilter(ReturnCode targetRetCode) { + this.targetRetCode = targetRetCode; + } + + @Override + public ReturnCode filterKeyValue(Cell v) throws IOException { + this.didCellPassToTheFilter = true; + return targetRetCode; + } + } + + @Test + public void testShouldPassCurrentCellToFilter() throws IOException { + KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, + Bytes.toBytes("value")); + KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 2, + Bytes.toBytes("value")); + KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 3, + Bytes.toBytes("value")); + KeyValue kv4 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4, + Bytes.toBytes("value")); + + MockFilter mockFilter = new MockFilter(ReturnCode.NEXT_COL); + FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter); + + filter.filterKeyValue(kv1); + assertTrue(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv2); + assertFalse(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv3); + assertTrue(mockFilter.didCellPassToTheFilter); + + mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL); + filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter); + + filter.filterKeyValue(kv1); + assertTrue(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv2); + assertFalse(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv3); + assertTrue(mockFilter.didCellPassToTheFilter); + + mockFilter = new MockFilter(ReturnCode.NEXT_ROW); + filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter); + filter.filterKeyValue(kv1); + assertTrue(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv2); + assertFalse(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv3); + assertFalse(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv4); + assertTrue(mockFilter.didCellPassToTheFilter); + + mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW); + filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter); + filter.filterKeyValue(kv1); + assertTrue(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv2); + assertFalse(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv3); + assertFalse(mockFilter.didCellPassToTheFilter); + + mockFilter.didCellPassToTheFilter = false; + filter.filterKeyValue(kv4); + assertTrue(mockFilter.didCellPassToTheFilter); + } + + @Test + public void testTheMaximalRule() throws IOException { + KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, + Bytes.toBytes("value")); + MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE); + MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL); + MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW); + MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL); + MockFilter filter5 = new MockFilter(ReturnCode.SKIP); + MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT); + MockFilter filter7 = new MockFilter(ReturnCode.NEXT_ROW); + + FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2); + assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, filter2, filter3); + assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter5, filter6); + assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter6); + assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter1); + assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter2, filter1, filter5); + assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1)); + filterList = new FilterList(Operator.MUST_PASS_ALL, filter2, + new FilterList(Operator.MUST_PASS_ALL, filter3, filter4)); + assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter7); + assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1)); + } + + @Test + public void testTheMinimalRule() throws IOException { + KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, + Bytes.toBytes("value")); + MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE); + MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL); + MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW); + MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL); + MockFilter filter5 = new MockFilter(ReturnCode.SKIP); + MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT); + FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2); + assertEquals(filterList.filterKeyValue(kv1), ReturnCode.INCLUDE); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, filter3); + assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter5, filter6); + assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter6); + assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter1); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter2, filter1, filter5); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, + new FilterList(Operator.MUST_PASS_ONE, filter3, filter4)); + assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, + new FilterList(Operator.MUST_PASS_ONE, filter3, filter4)); + assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1)); + + filterList = new FilterList(Operator.MUST_PASS_ONE, filter6, filter6); + assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1)); + } + + private static class MockNextRowFilter extends FilterBase { + private int hitCount = 0; + + public ReturnCode filterKeyValue(Cell v) throws IOException { + hitCount++; + return ReturnCode.NEXT_ROW; + } + + public int getHitCount() { + return hitCount; + } + } + + @Test + public void testRowCountFilter() throws IOException { + KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam1"), Bytes.toBytes("a"), 1, + Bytes.toBytes("value")); + KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam2"), Bytes.toBytes("a"), 2, + Bytes.toBytes("value")); + MockNextRowFilter mockNextRowFilter = new MockNextRowFilter(); + FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockNextRowFilter); + filter.filterKeyValue(kv1); + filter.filterKeyValue(kv2); + assertEquals(2, mockNextRowFilter.getHitCount()); + } + + @Test + public void testKeyOnlyFilterTransformCell() throws IOException { + Cell c; + KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), + 1, Bytes.toBytes("value1")); + KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), + 2, Bytes.toBytes("value2")); + + Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"), + CompareOp.EQUAL, Bytes.toBytes("value1")); + Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"), + CompareOp.EQUAL, Bytes.toBytes("value2")); + FilterList internalFilterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2); + + FilterList keyOnlyFilterFirst = + new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList); + + assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterKeyValue(kv1)); + c = keyOnlyFilterFirst.transformCell(kv1); + assertEquals(0, c.getValueLength()); + assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterKeyValue(kv2)); + c = keyOnlyFilterFirst.transformCell(kv2); + assertEquals(0, c.getValueLength()); + + internalFilterList.reset(); + FilterList keyOnlyFilterLast = + new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList); + assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterKeyValue(kv1)); + c = keyOnlyFilterLast.transformCell(kv1); + assertEquals(0, c.getValueLength()); + assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterKeyValue(kv2)); + c = keyOnlyFilterLast.transformCell(kv2); + assertEquals(0, c.getValueLength()); + } + + @Test + public void testEmptyFilterListTransformCell() throws IOException { + KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), + 1, Bytes.toBytes("value")); + FilterList filterList = new FilterList(Operator.MUST_PASS_ALL); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv)); + assertEquals(kv, filterList.transformCell(kv)); + + filterList = new FilterList(Operator.MUST_PASS_ONE); + assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv)); + assertEquals(kv, filterList.transformCell(kv)); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java index dd2399f..2a13ac8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java @@ -36,10 +36,10 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TestName; /** - * Tests filter Lists in ways that rely on a MiniCluster. - * Where possible, favor tests in TestFilterList and TestFilterFromRegionSide instead. + * Tests filter Lists in ways that rely on a MiniCluster. Where possible, favor tests in + * TestFilterList and TestFilterFromRegionSide instead. */ -@Category({MediumTests.class, FilterTests.class}) +@Category({ MediumTests.class, FilterTests.class }) public class TestFilterListOnMini { private static final Log LOG = LogFactory.getLog(TestFilterListOnMini.class); -- 2.7.4