+ * NOTE: associations list do not encode the counting list data. You
+ * should use {@link CountingListBuilder} to build that information and then
+ * merge the results of both {@link #build(IntsRef, Iterable)}.
+ */
+public class AssociationsListBuilder implements CategoryListBuilder {
+
+ private final CategoryAssociationsContainer associations;
+ private final ByteArrayDataOutput output = new ByteArrayDataOutput();
+
+ public AssociationsListBuilder(CategoryAssociationsContainer associations) {
+ this.associations = associations;
+ }
+
+ @Override
+ public Map
+ * NOTE: you are not expected to modify the returned map.
*/
- public float getAssociation(int ordinal) {
- if (ordinalAssociations.containsKey(ordinal)) {
- return ordinalAssociations.get(ordinal);
- }
-
- return Float.NaN;
+ public IntToFloatMap getAssociations(int docID) throws IOException {
+ ordinalAssociations.clear();
+ return setNextDoc(docID) ? ordinalAssociations : null;
}
-
+
}
Index: lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java (working copy)
@@ -31,12 +31,6 @@
private final IntToIntMap ordinalAssociations = new IntToIntMap();
- /**
- * The long-special-value returned for ordinals which have no associated int
- * value. It is not in the int range of values making it a valid mark.
- */
- public final static long NO_ASSOCIATION = Integer.MAX_VALUE + 1;
-
public IntAssociationsPayloadIterator(IndexReader reader, String field, CategoryIntAssociation association)
throws IOException {
super(reader, field, association);
@@ -47,22 +41,16 @@
ordinalAssociations.put(ordinal, association.getValue());
}
- @Override
- public boolean setNextDoc(int docId) throws IOException {
- ordinalAssociations.clear();
- return super.setNextDoc(docId);
- }
-
/**
- * Get the integer association value for the given ordinal, or
- * {@link #NO_ASSOCIATION} in case the ordinal has no association value.
+ * Returns the integer association values of the categories that are
+ * associated with the given document, or {@code null} if the document has no
+ * associations.
+ *
+ * NOTE: you are not expected to modify the returned map.
*/
- public long getAssociation(int ordinal) {
- if (ordinalAssociations.containsKey(ordinal)) {
- return ordinalAssociations.get(ordinal);
- }
-
- return NO_ASSOCIATION;
+ public IntToIntMap getAssociations(int docID) throws IOException {
+ ordinalAssociations.clear();
+ return setNextDoc(docID) ? ordinalAssociations : null;
}
}
Index: lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java (working copy)
@@ -1,19 +1,11 @@
package org.apache.lucene.facet.index;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map.Entry;
+import java.util.Map;
-import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.util.PartitionsUtils;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.UnsafeByteArrayOutputStream;
-import org.apache.lucene.util.encoding.IntEncoder;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -33,149 +25,14 @@
*/
/**
- * Builds a category list by encoding the category ordinals into one or more
- * {@link BytesRef}. Each {@link BytesRef} corresponds to a set of ordinals that
- * belong to the same partition. When partitions are not enabled (i.e.
- * {@link FacetIndexingParams#getPartitionSize()} returns
- * {@link Integer#MAX_VALUE}), only one {@link BytesRef} is returned by this
- * class.
+ * Builds a category list data by encoding the appropriate information for every
+ * category and ordinal given to {@link #build(IntsRef, Iterable)}.
+ *
+ * @lucene.experimental
*/
-public class CategoryListBuilder {
-
- /** Specializes encoding ordinals when partitions are enabled/disabled. */
- private static abstract class OrdinalsEncoder {
- OrdinalsEncoder() {}
- public abstract void encode(int ordinal);
- public abstract HashMap
+ * Counting lists are used usually for computing the weight of categories by
+ * summing their number of occurrences (hence counting) in a result set.
+ */
+public class CountingListBuilder implements CategoryListBuilder {
+
+ /** Specializes encoding ordinals when partitions are enabled/disabled. */
+ private static abstract class OrdinalsEncoder {
+ OrdinalsEncoder() {}
+ public abstract Map
+ * NOTE: the {@code ordinals} array is modified by adding parent
+ * ordinals to it. Also, some encoders may sort the array and remove duplicate
+ * ordinals. Therefore you may want to invoke this method after you finished
+ * processing the array for other purposes.
+ */
+ @Override
+ public Map
- * By default, all categories are written to the same category list, but
- * applications which know in advance that in some situations only parts of
- * the category hierarchy needs to be counted can divide the categories into
- * two or more different category lists.
- *
- * If {@code null} is returned for a category, it means that this category
- * should not appear in any category list, and thus weights for it cannot be
- * aggregated. This category can still be used for drill-down, even though the
- * its weight is unknown.
+ * Returns the {@link CategoryListParams} for this {@link CategoryPath}. The
+ * default implementation returns the same {@link CategoryListParams} for all
+ * categories (even if {@code category} is {@code null}).
*
* @see PerDimensionIndexingParams
*/
Index: lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java (working copy)
@@ -78,7 +78,9 @@
/**
* Returns the {@link CategoryListParams} for the corresponding dimension
- * which is returned by {@code category.getComponent(0)}.
+ * which is returned by {@code category.getComponent(0)}. If {@code category}
+ * is {@code null}, or was not specified in the map given to the constructor,
+ * returns the default {@link CategoryListParams}.
*/
@Override
public CategoryListParams getCategoryListParams(CategoryPath category) {
Index: lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java (revision 0)
+++ lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java (working copy)
@@ -0,0 +1,52 @@
+package org.apache.lucene.facet.search;
+
+import java.io.IOException;
+
+import org.apache.lucene.util.IntsRef;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * An interface for obtaining the category ordinals of documents.
+ *
+ * NOTE: this class operates as a key to a map, and therefore you should
+ * implement {@code equals()} and {@code hashCode()} for proper behavior.
+ *
+ * @lucene.experimental
+ */
+public interface CategoryListIterator {
+
+ /**
+ * Initializes the iterator. This method must be called before any calls to
+ * {@link #getOrdinals(int, IntsRef)}, and its return value indicates whether there are
+ * any relevant documents for this iterator.
+ */
+ public boolean init() throws IOException;
+
+ /**
+ * Stores the category ordinals of the given document ID in the given
+ * {@link IntsRef}, starting at position 0 upto {@link IntsRef#length}. Grows
+ * the {@link IntsRef} if it is not large enough.
+ *
+ *
+ * NOTE: if the requested document does not category ordinals
+ * associated with it, {@link IntsRef#length} is set to zero.
+ */
+ public void getOrdinals(int docID, IntsRef ints) throws IOException;
+
+}
Property changes on: lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java (revision 0)
+++ lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java (working copy)
@@ -0,0 +1,83 @@
+package org.apache.lucene.facet.search;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.encoding.IntDecoder;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A {@link CategoryListIterator} which reads the category ordinals from a
+ * payload.
+ *
+ * @lucene.experimental
+ */
+public class PayloadCategoryListIteraor implements CategoryListIterator {
+
+ private final IntDecoder decoder;
+ private final IndexReader indexReader;
+ private final Term term;
+ private final PayloadIterator pi;
+ private final int hashCode;
+
+ public PayloadCategoryListIteraor(IndexReader indexReader, Term term, IntDecoder decoder) throws IOException {
+ pi = new PayloadIterator(indexReader, term);
+ this.decoder = decoder;
+ hashCode = indexReader.hashCode() ^ term.hashCode();
+ this.term = term;
+ this.indexReader = indexReader;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof PayloadCategoryListIteraor)) {
+ return false;
+ }
+ PayloadCategoryListIteraor that = (PayloadCategoryListIteraor) other;
+ if (hashCode != that.hashCode) {
+ return false;
+ }
+
+ // Hash codes are the same, check equals() to avoid cases of hash-collisions.
+ return indexReader.equals(that.indexReader) && term.equals(that.term);
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ @Override
+ public boolean init() throws IOException {
+ return pi.init();
+ }
+
+ @Override
+ public void getOrdinals(int docID, IntsRef ints) throws IOException {
+ ints.length = 0;
+ BytesRef payload = pi.getPayload(docID);
+ if (payload != null) {
+ decoder.decode(payload, ints);
+ }
+ }
+
+}
Property changes on: lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java (working copy)
@@ -1,115 +0,0 @@
-package org.apache.lucene.facet.search;
-
-import java.io.IOException;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.UnsafeByteArrayInputStream;
-import org.apache.lucene.util.encoding.IntDecoder;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * A payload deserializer comes with its own working space (buffer). One need to
- * define the {@link IndexReader} and {@link Term} in which the payload resides.
- * The iterator then consumes the payload information of each document and
- * decodes it into categories. A typical use case of this class is:
- *
- *
- * Users should call this method with increasing docIds, and implementations
- * can assume that this is the case.
+ * Returns the {@link BytesRef payload} of the given document, or {@code null}
+ * if the document does not exist, there are no more documents in the posting
+ * list, or the document exists but has not payload. You should call
+ * {@link #init()} before the first call to this method.
*/
- public boolean setdoc(int docId) throws IOException {
+ public BytesRef getPayload(int docID) throws IOException {
if (!hasMore) {
- return false;
+ return null;
}
// re-basing docId->localDocID is done fewer times than currentDoc->globalDoc
- int localDocID = docId - curDocBase;
+ int localDocID = docID - curDocBase;
if (curDocID > localDocID) {
// document does not exist
- return false;
+ return null;
}
if (curDocID < localDocID) {
// look for the document either in that segment, or others
while (hasMore && (curDocID = currentDPE.advance(localDocID)) == DocIdSetIterator.NO_MORE_DOCS) {
nextSegment(); // also updates curDocID
- localDocID = docId - curDocBase;
+ localDocID = docID - curDocBase;
// nextSegment advances to nextDoc, so check if we still need to advance
if (curDocID >= localDocID) {
break;
@@ -127,7 +126,7 @@
// 1. we iterated over all segments (hasMore=false)
// 2. current segment advanced to a doc, either requested or higher
if (!hasMore || curDocID != localDocID) {
- return false;
+ return null;
}
}
@@ -135,12 +134,7 @@
assert currentDPE.freq() == 1 : "expecting freq=1 (got " + currentDPE.freq() + ") term=" + term + " doc=" + (curDocID + curDocBase);
int pos = currentDPE.nextPosition();
assert pos != -1 : "no positions for term=" + term + " doc=" + (curDocID + curDocBase);
- data = currentDPE.getPayload();
- return data != null;
+ return currentDPE.getPayload();
}
- public BytesRef getPayload() {
- return data;
- }
-
}
Index: lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java (working copy)
@@ -10,6 +10,7 @@
import java.util.logging.Logger;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.IntsRef;
import org.apache.lucene.facet.search.aggregator.Aggregator;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -231,9 +232,9 @@
facetArrays.free(); // to get a cleared array for this partition
}
- HashMap
- * Compatibility:
- * Extensions of this class need to implement {@link #encode(int)} in order to
- * build the proper indicator (flags). When enough values were accumulated
- * (typically the batch size), extensions can call {@link #encodeChunk()} to
- * flush the indicator and the rest of the values.
+ * Extensions of this class need to implement {@link #encode(IntsRef, BytesRef)}
+ * in order to build the proper indicator (flags). When enough values were
+ * accumulated (typically the batch size), extensions can call
+ * {@link #encodeChunk(BytesRef)} to flush the indicator and the rest of the
+ * values.
*
* NOTE: flags encoders do not accept values ≤ 0 (zero) in their
- * {@link #encode(int)}. For performance reasons they do not check that
- * condition, however if such value is passed the result stream may be corrupt
- * or an exception will be thrown. Also, these encoders perform the best when
- * there are many consecutive small values (depends on the encoder
+ * {@link #encode(IntsRef, BytesRef)}. For performance reasons they do not check
+ * that condition, however if such value is passed the result stream may be
+ * corrupt or an exception will be thrown. Also, these encoders perform the best
+ * when there are many consecutive small values (depends on the encoder
* implementation). If that is not the case, the encoder will occupy 1 more byte
* for every batch number of integers, over whatever
* {@link VInt8IntEncoder} would have occupied. Therefore make sure to check
* whether your data fits into the conditions of the specific encoder.
*
* For the reasons mentioned above, these encoders are usually chained with
- * {@link UniqueValuesIntEncoder} and {@link DGapIntEncoder} in the following
- * manner:
* NOTE: this encoder assumes the values are given to
- * {@link #encode(int)} in an ascending sorted manner, which ensures only
+ * {@link #encode(IntsRef, BytesRef)} in an ascending sorted manner, which ensures only
* positive values are encoded and thus yields better performance. If you are
* not sure whether the values are sorted or not, it is possible to chain this
* encoder with {@link SortingIntEncoder} to ensure the values will be
@@ -37,17 +37,20 @@
*/
public class DGapIntEncoder extends IntEncoderFilter {
- private int prev = 0;
-
/** Initializes with the given encoder. */
public DGapIntEncoder(IntEncoder encoder) {
super(encoder);
}
@Override
- public void encode(int value) throws IOException {
- encoder.encode(value - prev);
- prev = value;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ int prev = 0;
+ for (int i = values.offset; i < upto; i++) {
+ int tmp = values.ints[i];
+ values.ints[i] -= prev;
+ prev = tmp;
+ }
+ encoder.doEncode(values, buf, upto);
}
@Override
@@ -56,12 +59,6 @@
}
@Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- prev = 0;
- }
-
- @Override
public String toString() {
return "DGap (" + encoder.toString() + ")";
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,20 +21,17 @@
*/
/**
- * Decodes data which was encoded by {@link EightFlagsIntEncoder}. Scans
- * the
+ * NOTE: it is ok to use the buffer's offset as the current position in
+ * the buffer (and modify it), it will be reset by
+ * {@link #decode(BytesRef, IntsRef)}. returns.
+ */
+ protected abstract void doDecode(BytesRef buf, IntsRef values, int upto);
+
+ /**
+ * Called before {@link #doDecode(BytesRef, IntsRef, int)} so that decoders
+ * can reset their state.
+ */
+ protected void reset() {
+ // do nothing by default
}
/**
- * Decodes data received from the input stream, and returns one decoded
- * integer. If end of stream is reached, {@link #EOS} is returned.
- *
- * @return one decoded integer as long or {@link #EOS} if end-of-stream
- * reached.
- * @throws IOException if an I/O error occurs
+ * Decodes the values from the buffer into the given {@link IntsRef}. Note
+ * that {@code values'} offset and length are set to 0.
*/
- public abstract long decode() throws IOException;
+ public final void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0; // must do that because we cannot grow() them otherwise
+
+ // some decoders may use the buffer's offset as a position index, so save
+ // current offset.
+ int bufOffset = buf.offset;
+
+ reset();
+ doDecode(buf, values, buf.offset + buf.length);
+ assert values.offset == 0 : "offset should not have been modified by the decoder.";
+
+ // fix offset
+ buf.offset = bufOffset;
+ }
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java (working copy)
@@ -1,8 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,94 +21,47 @@
*/
/**
- * Encodes integers to a set {@link OutputStream}. Extending classes need to
- * override {@link #encode(int)} to encode the value using their encoding
- * algorithm. The default implementation of {@link #close()} closes the set
- * {@link OutputStream}.
+ * Encodes integers to a set {@link BytesRef}. Extending classes need to
+ * override {@link #encode(IntsRef, BytesRef)} to encode the values using their encoding
+ * algorithm.
*
- * The default {@link #IntEncoder() constructor} is provided for convenience
- * only. One must call {@link #reInit(OutputStream)} before calling
- * {@link #encode(int)} or {@link #close()}.
- *
* For convenience, each encoder implements {@link #createMatchingDecoder()} for
* easy access to the matching decoder.
- *
- * NOTE: some implementations may buffer the encoded values in memory
- * (such as {@link IntEncoderFilter} implementations) and encoding will happen
- * only upon calling {@link #close()}. Therefore it is important to always call
- * {@link #close()} on the encoder at hand.
- *
- * NOTE: encoders are usually not thread safe, unless specifically
- * documented otherwise by an implementation.
*
* @lucene.experimental
*/
-public abstract class IntEncoder implements Closeable {
+public abstract class IntEncoder {
- protected OutputStream out = null;
+ public IntEncoder() {}
/**
- * Default constructor, provided here for robustness: if in the future a
- * constructor with parameters will be added, this might break custom
- * implementations of this class which call this implicit constructor. So we
- * make it explicit to avoid any such issue in the future.
+ * Performs the actual encoding. Values should be read from
+ * {@link IntsRef#offset} up to {@code upto}. Also, it is guaranteed that
+ * {@code buf's} offset and length are set to 0 and the encoder is expected to
+ * update {@link BytesRef#length}, but not {@link BytesRef#offset}.
*/
- public IntEncoder() {
- }
-
+ protected abstract void doEncode(IntsRef values, BytesRef buf, int upto);
+
/**
- * Instructs the encoder to finish the encoding process. This method closes
- * the output stream which was specified by {@link #reInit(OutputStream)
- * reInit}. An implementation may do here additional cleanup required to
- * complete the encoding, such as flushing internal buffers, etc.
- * NOTE: overriding classes should make sure they either call
- *
- * NOTE: this method should create a new instance of the matching
- * decoder and leave the instance sharing to the caller. Returning the same
- * instance over and over is risky because encoders and decoders are not
- * thread safe.
- */
+ /** Returns an {@link IntDecoder} which matches this encoder. */
public abstract IntDecoder createMatchingDecoder();
- /**
- * Reinitializes the encoder with the give {@link OutputStream}. For
- * re-usability it can be changed without the need to reconstruct a new
- * object.
- *
- * NOTE: after calling {@link #close()}, one must call
- * this method even if the output stream itself hasn't changed. An example
- * case is that the output stream wraps a byte[], and the output stream itself
- * is reset, but its instance hasn't changed. Some implementations of
- * {@link IntEncoder} may write some metadata about themselves to the output
- * stream, and therefore it is imperative that one calls this method before
- * encoding any data.
- */
- public void reInit(OutputStream out) {
- this.out = out;
- }
-
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java (working copy)
@@ -1,7 +1,5 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,17 +19,7 @@
*/
/**
- * An abstract implementation of {@link IntEncoder} which is served as a filter
- * on the values to encode. An encoder filter wraps another {@link IntEncoder}
- * which does the actual encoding. This allows for chaining filters and
- * encoders, such as:
- * The default implementation implements {@link #close()} by closing the wrapped
- * encoder and {@link #reInit(OutputStream)} by re-initializing the wrapped
- * encoder.
+ * An abstract implementation of {@link IntEncoder} which wraps another encoder.
*
* @lucene.experimental
*/
@@ -44,15 +32,8 @@
}
@Override
- public void close() throws IOException {
- // There is no need to call super.close(), since we don't pass the output
- // stream to super.
- encoder.close();
+ public void reset() {
+ encoder.reset();
}
- @Override
- public void reInit(OutputStream out) {
- encoder.reInit(out);
- }
-
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,57 +21,68 @@
*/
/**
- * Decodes data which was encoded by {@link NOnesIntEncoder}. Uses a
- * {@link FourFlagsIntDecoder} to perform the actual encoding and translates the
- * values back as described in {@link NOnesIntEncoder}.
+ * Decodes values encoded encoded with {@link NOnesIntEncoder}.
*
- * @see NOnesIntEncoder
* @lucene.experimental
*/
public class NOnesIntDecoder extends FourFlagsIntDecoder {
- /** Number of consecutive '1's to generate upon decoding a '2'. */
- private int n;
-
- private int onesCounter;
-
+ // Number of consecutive '1's to generate upon decoding a '2'
+ private final int n;
+ private final IntsRef internalBuffer;
+
/**
* Constructs a decoder with a given N (Number of consecutive '1's which are
* translated into a single target value '2'.
*/
public NOnesIntDecoder(int n) {
this.n = n;
+ // initial size (room for 100 integers)
+ internalBuffer = new IntsRef(100);
}
@Override
- public long decode() throws IOException {
- // If we read '2', we should return n '1's.
- if (onesCounter > 0) {
- --onesCounter;
- return 1;
+ protected void reset() {
+ internalBuffer.length = 0;
+ super.reset();
+ }
+
+ @Override
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ super.doDecode(buf, internalBuffer, upto);
+ if (values.ints.length < internalBuffer.length) {
+ // need space for internalBuffer.length to internalBuffer.length*N,
+ // grow mildly at first
+ values.grow(internalBuffer.length * n/2);
}
-
- long decode = super.decode();
- if (decode == 1) {
- return 1;
+
+ for (int i = 0; i < internalBuffer.length; i++) {
+ int decode = internalBuffer.ints[i];
+ if (decode == 1) {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
+ }
+ // 1 is 1
+ values.ints[values.length++] = 1;
+ } else if (decode == 2) {
+ if (values.length + n >= values.ints.length) {
+ values.grow(values.length + n); // grow by few items, however not too many
+ }
+ // '2' means N 1's
+ for (int j = 0; j < n; j++) {
+ values.ints[values.length++] = 1;
+ }
+ } else {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
+ }
+ // any other value is val-1
+ values.ints[values.length++] = decode - 1;
+ }
}
- if (decode == 2) {
- onesCounter = n - 1;
- return 1;
- }
- if (decode == 3) {
- return 2;
- }
- return decode == EOS ? EOS : decode - 1;
}
@Override
- public void reInit(InputStream in) {
- super.reInit(in);
- onesCounter = 0;
- }
-
- @Override
public String toString() {
return "NOnes (" + n + ") (" + super.toString() + ")";
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -50,50 +50,59 @@
*/
public class NOnesIntEncoder extends FourFlagsIntEncoder {
+ private final IntsRef internalBuffer;
+
/** Number of consecutive '1's to be translated into single target value '2'. */
- private int n;
+ private final int n;
- /** Counts the number of consecutive ones seen. */
- private int onesCounter = 0;
-
/**
* Constructs an encoder with a given value of N (N: Number of consecutive
* '1's to be translated into single target value '2').
*/
public NOnesIntEncoder(int n) {
this.n = n;
+ internalBuffer = new IntsRef(n);
}
@Override
- public void close() throws IOException {
- // We might have ones in our buffer, encode them as neccesary.
- while (onesCounter-- > 0) {
- super.encode(1);
- }
-
- super.close();
+ protected void reset() {
+ internalBuffer.length = 0;
+ super.reset();
}
-
+
@Override
- public void encode(int value) throws IOException {
- if (value == 1) {
- // Increment the number of consecutive ones seen so far
- if (++onesCounter == n) {
- super.encode(2);
- onesCounter = 0;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ // make sure the internal buffer is large enough
+ if (values.length > internalBuffer.ints.length) {
+ internalBuffer.grow(values.length);
+ }
+
+ int onesCounter = 0;
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ if (value == 1) {
+ // every N 1's should be encoded as '2'
+ if (++onesCounter == n) {
+ internalBuffer.ints[internalBuffer.length++] = 2;
+ onesCounter = 0;
+ }
+ } else {
+ // there might have been 1's that we need to encode
+ while (onesCounter > 0) {
+ --onesCounter;
+ internalBuffer.ints[internalBuffer.length++] = 1;
+ }
+
+ // encode value as value+1
+ internalBuffer.ints[internalBuffer.length++] = value + 1;
}
- return;
}
-
- // If it's not one - there might have been ones we had to encode prior to
- // this value
+ // there might have been 1's that we need to encode
while (onesCounter > 0) {
--onesCounter;
- super.encode(1);
+ internalBuffer.ints[internalBuffer.length++] = 1;
}
-
- // encode value + 1 --> the translation.
- super.encode(value + 1);
+ super.doEncode(internalBuffer, buf, internalBuffer.length);
}
@Override
@@ -102,12 +111,6 @@
}
@Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- onesCounter = 0;
- }
-
- @Override
public String toString() {
return "NOnes (" + n + ") (" + super.toString() + ")";
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.StreamCorruptedException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,41 +21,24 @@
*/
/**
- * A simple stream decoder which can decode values encoded with
- * {@link SimpleIntEncoder}.
+ * Decodes values encoded with {@link SimpleIntEncoder}.
*
* @lucene.experimental
*/
public class SimpleIntDecoder extends IntDecoder {
- /**
- * reusable buffer - allocated only once as this is not a thread-safe object
- */
- private byte[] buffer = new byte[4];
-
@Override
- public long decode() throws IOException {
-
- // we need exactly 4 bytes to decode an int in this decoder impl, otherwise, throw an exception
- int offset = 0;
- while (offset < 4) {
- int nRead = in.read(buffer, offset, 4 - offset);
- if (nRead == -1) {
- if (offset > 0) {
- throw new StreamCorruptedException(
- "Need 4 bytes for decoding an int, got only " + offset);
- }
- return EOS;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
}
- offset += nRead;
+ values.ints[values.length++] =
+ ((buf.bytes[buf.offset++] & 0xFF) << 24) |
+ ((buf.bytes[buf.offset++] & 0xFF) << 16) |
+ ((buf.bytes[buf.offset++] & 0xFF) << 8) |
+ (buf.bytes[buf.offset++] & 0xFF);
}
-
- int v = buffer[3] & 0xff;
- v |= (buffer[2] << 8) & 0xff00;
- v |= (buffer[1] << 16) & 0xff0000;
- v |= (buffer[0] << 24) & 0xff000000;
-
- return v;
}
@Override
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java (working copy)
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -26,22 +27,21 @@
*/
public class SimpleIntEncoder extends IntEncoder {
- /**
- * This method makes sure the value wasn't previously encoded by checking
- * against the Set. If the value wasn't encoded, it's added to the Set, and
- * encoded with {#link Vint8#encode}
- *
- * @param value
- * an integer to be encoded
- * @throws IOException
- * possibly thrown by the OutputStream
- */
@Override
- public void encode(int value) throws IOException {
- out.write(value >>> 24);
- out.write((value >> 16) & 0xFF);
- out.write((value >> 8) & 0xFF);
- out.write(value & 0xFF);
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ // ensure there's enough room in the buffer
+ int bytesNeeded = values.length * 4;
+ if (buf.bytes.length < bytesNeeded) {
+ buf.grow(bytesNeeded);
+ }
+
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ buf.bytes[buf.length++] = (byte) (value >>> 24);
+ buf.bytes[buf.length++] = (byte) ((value >> 16) & 0xFF);
+ buf.bytes[buf.length++] = (byte) ((value >> 8) & 0xFF);
+ buf.bytes[buf.length++] = (byte) (value & 0xFF);
+ }
}
@Override
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java (working copy)
@@ -1,9 +1,10 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
import java.util.Arrays;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -23,61 +24,29 @@
/**
* An {@link IntEncoderFilter} which sorts the values to encode in ascending
- * order before encoding them. Encoding therefore happens upon calling
- * {@link #close()}. Since this encoder is usually chained with another encoder
- * that relies on sorted values, it does not offer a default constructor.
+ * order before encoding them.
*
* @lucene.experimental
*/
public class SortingIntEncoder extends IntEncoderFilter {
- private float grow = 2.0f;
- private int index = 0;
- private int[] set = new int[1024];
-
/** Initializes with the given encoder. */
public SortingIntEncoder(IntEncoder encoder) {
super(encoder);
}
@Override
- public void close() throws IOException {
- if (index == 0) {
- return;
- }
-
- Arrays.sort(set, 0, index);
- for (int i = 0; i < index; i++) {
- encoder.encode(set[i]);
- }
- encoder.close();
- index = 0;
-
- super.close();
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ Arrays.sort(values.ints, values.offset, upto);
+ encoder.doEncode(values, buf, upto);
}
@Override
- public void encode(int value) throws IOException {
- if (index == set.length) {
- int[] newSet = new int[(int) (set.length * grow)];
- System.arraycopy(set, 0, newSet, 0, set.length);
- set = newSet;
- }
- set[index++] = value;
- }
-
- @Override
public IntDecoder createMatchingDecoder() {
return encoder.createMatchingDecoder();
}
@Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- index = 0;
- }
-
- @Override
public String toString() {
return "Sorting (" + encoder.toString() + ")";
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,7 +22,7 @@
/**
* An {@link IntEncoderFilter} which ensures only unique values are encoded. The
- * implementation assumes the values given to {@link #encode(int)} are sorted.
+ * implementation assumes the values given to {@link #encode(IntsRef, BytesRef)} are sorted.
* If this is not the case, you can chain this encoder with
* {@link SortingIntEncoder}.
*
@@ -30,26 +30,23 @@
*/
public final class UniqueValuesIntEncoder extends IntEncoderFilter {
- /**
- * Denotes an illegal value which we can use to init 'prev' to. Since all
- * encoded values are integers, this value is init to MAX_INT+1 and is of type
- * long. Therefore we are guaranteed not to get this value in encode.
- */
- private static final long ILLEGAL_VALUE = Integer.MAX_VALUE + 1;
-
- private long prev = ILLEGAL_VALUE;
-
/** Constructs a new instance with the given encoder. */
public UniqueValuesIntEncoder(IntEncoder encoder) {
super(encoder);
}
@Override
- public void encode(int value) throws IOException {
- if (prev != value) {
- encoder.encode(value);
- prev = value;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ int prev = values.ints[values.offset];
+ int idx = values.offset + 1;
+ for (int i = idx; i < upto; i++) {
+ if (values.ints[i] != prev) {
+ values.ints[idx++] = values.ints[i];
+ prev = values.ints[i];
+ }
}
+ values.length = idx - values.offset;
+ encoder.doEncode(values, buf, idx);
}
@Override
@@ -58,12 +55,6 @@
}
@Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- prev = ILLEGAL_VALUE;
- }
-
- @Override
public String toString() {
return "Unique (" + encoder.toString() + ")";
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java (revision 0)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java (working copy)
@@ -0,0 +1,153 @@
+package org.apache.lucene.util.encoding;
+
+import org.apache.lucene.util.BytesRef;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Variable-length encoding of 32-bit integers, into 8-bit bytes. A number is
+ * encoded as follows:
+ * An encoder encodes the integers that are passed to {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} into a
-set output stream (see {@link
-org.apache.lucene.util.encoding.IntEncoder#reInit(OutputStream)
-reInit}). One should always call {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close} when all
-integers have been encoded, to ensure proper finish by the encoder. Some
-encoders buffer values in-memory and encode in batches in order to
-optimize the encoding, and not closing them may result in loss of
-information or corrupt stream.
- A proper and typical usage of an encoder looks like this:
- Some encoders don't perform any encoding at all, or do not include an
+
+Some encoders don't perform any encoding at all, or do not include an
encoding logic. Those are called {@link
org.apache.lucene.util.encoding.IntEncoderFilter}s. A filter is an
encoder which delegates the encoding task to a given encoder, however
@@ -76,91 +35,6 @@
which encodes the gaps between values rather than the values themselves.
Another example is {@link
org.apache.lucene.util.encoding.SortingIntEncoder} which sorts all the
-values in ascending order before they are sent for encoding. This
-encoder aggregates the values in its {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} implementation
-and decoding only happens upon calling {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close}.
-
- * IndexReader reader = [open your reader];
- * Term t = new Term("field", "where-payload-exists");
- * CategoryListIterator cli = new PayloadIntDecodingIterator(reader, t);
- * if (!cli.init()) {
- * // it means there are no payloads / documents associated with that term.
- * // Usually a sanity check. However, init() must be called.
- * }
- * DocIdSetIterator disi = [you usually iterate on something else, such as a Scorer];
- * int doc;
- * while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- * cli.setdoc(doc);
- * long category;
- * while ((category = cli.nextCategory()) < Integer.MAX_VALUE) {
- * }
- * }
- *
- *
- * @lucene.experimental
- */
-public class PayloadIntDecodingIterator implements CategoryListIterator {
-
- private final UnsafeByteArrayInputStream ubais;
- private final IntDecoder decoder;
-
- private final IndexReader indexReader;
- private final Term term;
- private final PayloadIterator pi;
- private final int hashCode;
-
- public PayloadIntDecodingIterator(IndexReader indexReader, Term term, IntDecoder decoder) throws IOException {
- pi = new PayloadIterator(indexReader, term);
- ubais = new UnsafeByteArrayInputStream();
- this.decoder = decoder;
- hashCode = indexReader.hashCode() ^ term.hashCode();
- this.term = term;
- this.indexReader = indexReader;
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof PayloadIntDecodingIterator)) {
- return false;
- }
- PayloadIntDecodingIterator that = (PayloadIntDecodingIterator) other;
- if (hashCode != that.hashCode) {
- return false;
- }
-
- // Hash codes are the same, check equals() to avoid cases of hash-collisions.
- return indexReader.equals(that.indexReader) && term.equals(that.term);
- }
-
- @Override
- public int hashCode() {
- return hashCode;
- }
-
- @Override
- public boolean init() throws IOException {
- return pi.init();
- }
-
- @Override
- public long nextCategory() throws IOException {
- return decoder.decode();
- }
-
- @Override
- public boolean skipTo(int docId) throws IOException {
- if (!pi.setdoc(docId)) {
- return false;
- }
-
- // Initializing the decoding mechanism with the new payload data
- BytesRef data = pi.getPayload();
- ubais.reInit(data.bytes, data.offset, data.length + data.offset);
- decoder.reInit(ubais);
- return true;
- }
-
-}
Index: lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (working copy)
@@ -34,9 +34,9 @@
* A utility class for iterating through a posting list of a given term and
* retrieving the payload of the first position in every document. For
* efficiency, this class does not check if documents passed to
- * {@link #setdoc(int)} are deleted, since it is usually used to iterate on
+ * {@link #getPayload(int)} are deleted, since it is usually used to iterate on
* payloads of documents that matched a query. If you need to skip over deleted
- * documents, you should do so before calling {@link #setdoc(int)}.
+ * documents, you should do so before calling {@link #getPayload(int)}.
*
* @lucene.experimental
*/
@@ -84,8 +84,8 @@
/**
* Initialize the iterator. Should be done before the first call to
- * {@link #setdoc(int)}. Returns {@code false} if no category list is found,
- * or the category list has no documents.
+ * {@link #getPayload(int)}. Returns {@code false} if no category list is
+ * found, or the category list has no documents.
*/
public boolean init() throws IOException {
nextSegment();
@@ -93,30 +93,29 @@
}
/**
- * Skip forward to document docId. Return true if this document exists and
- * has any payload.
- *
- *
- * Examples:
- *
- *
- * This encoder/decoder will correctly handle any 32-bit integer, but for negative numbers,
- * and positive numbers with more than 28 significant bits, encoding requires 5 bytes; this
- * is not an efficient encoding scheme for large
- * positive numbers or any negative number.
- *
- * This class has been used in products that have shipped to customers, and is needed to
- * decode legacy data. Do not modify this class in ways that will break compatibility.
- *
- * @lucene.experimental
- */
-public class Vint8 {
-
- /**
- * Because Java lacks call-by-reference, this class boxes the decoding position, which
- * is initially set by the caller, and returned after decoding, incremented by the number
- * of bytes processed.
- */
- public static class Position {
- /**
- * Creates a position value set to zero.
- */
- public Position() {
- // The initial position is zero by default.
- }
- /**
- * Creates a position set to {@code initialPosition}.
- * @param initialPosition The starting decoding position in the source buffer.
- */
- public Position(int initialPosition) {
- this.pos = initialPosition;
- }
- /**
- * The value passed by reference.
- */
- public int pos;
- }
-
- /**
- * Returns the number of bytes needed to encode {@code number}.
- * @param number The number whose encoded length is needed.
- * @return The number of bytes needed to encode {@code number}.
- */
- public static int bytesNeeded(int number) {
- if ((number & ~0x7F) == 0) {
- return 1;
- } else if ((number & ~0x3FFF) == 0) {
- return 2;
- } else if ((number & ~0x1FFFFF) == 0) {
- return 3;
- } else if ((number & ~0xFFFFFFF) == 0) {
- return 4;
- } else {
- return 5;
- }
- }
-
- /**
- * The maximum number of bytes needed to encode a number using {@code Vint8}.
- */
- public static final int MAXIMUM_BYTES_NEEDED = 5;
-
- /**
- * Encodes {@code number} to {@code out}.
- * @param number The value to be written in encoded form, to {@code out}.
- * @param out The output stream receiving the encoded bytes.
- * @exception IOException If there is a problem writing to {@code out}.
- */
- public static void encode(int number, OutputStream out) throws IOException {
- if ((number & ~0x7F) == 0) {
- out.write(number);
- } else if ((number & ~0x3FFF) == 0) {
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- } else if ((number & ~0x1FFFFF) == 0) {
- out.write(0x80 | (number >> 14));
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- } else if ((number & ~0xFFFFFFF) == 0) {
- out.write(0x80 | (number >> 21));
- out.write(0x80 | (number >> 14));
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- } else {
- out.write(0x80 | (number >> 28));
- out.write(0x80 | (number >> 21));
- out.write(0x80 | (number >> 14));
- out.write(0x80 | (number >> 7));
- out.write(0x7F & number);
- }
- }
-
- /**
- * Encodes {@code number} into {@code dest}, starting at offset {@code start} from
- * the beginning of the array. This method assumes {@code dest} is large enough to
- * hold the required number of bytes.
- * @param number The number to be encoded.
- * @param dest The destination array.
- * @param start The starting offset in the array.
- * @return The number of bytes used in the array.
- */
- public static int encode(int number, byte[] dest, int start) {
- if ((number & ~0x7F) == 0) {
- dest[start] = (byte) number;
- return 1;
- } else if ((number & ~0x3FFF) == 0) {
- dest[start] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 1] = (byte) (number & 0x7F);
- return 2;
- } else if ((number & ~0x1FFFFF) == 0) {
- dest[start] = (byte) (0x80 | ((number & 0x1FC000) >> 14));
- dest[start + 1] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 2] = (byte) (number & 0x7F);
- return 3;
- } else if ((number & ~0xFFFFFFF) == 0) {
- dest[start] = (byte) (0x80 | ((number & 0xFE00000) >> 21));
- dest[start + 1] = (byte) (0x80 | ((number & 0x1FC000) >> 14));
- dest[start + 2] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 3] = (byte) (number & 0x7F);
- return 4;
- } else {
- dest[start] = (byte) (0x80 | ((number & 0xF0000000) >> 28));
- dest[start + 1] = (byte) (0x80 | ((number & 0xFE00000) >> 21));
- dest[start + 2] = (byte) (0x80 | ((number & 0x1FC000) >> 14));
- dest[start + 3] = (byte) (0x80 | ((number & 0x3F80) >> 7));
- dest[start + 4] = (byte) (number & 0x7F);
- return 5;
- }
- }
-
- /**
- * Decodes a 32-bit integer from {@code bytes}, beginning at offset {@code pos.pos}.
- * The decoded value is returned, and {@code pos.pos} is incremented by the number of
- * bytes processed.
- * @param bytes The byte array containing an encoded value.
- * @param pos On entry, the starting position in the array; on return, one greater
- * than the position of the last byte decoded in the call.
- * @return The decoded value.
- */
- public static int decode(byte[] bytes, Position pos) {
- int value = 0;
- while (true) {
- byte first = bytes[pos.pos];
- ++pos.pos;
- value |= first & 0x7F;
- if ((first & 0x80) == 0) {
- return value;
- }
- value <<= 7;
- }
- }
-
- /**
- * Decodes a 32-bit integer from bytes read from {@code in}. Bytes are read,
- * one at a time, from {@code in}, and it is assumed they represent a 32-bit
- * integer encoded using this class's encoding scheme. The decoded value is
- * returned.
- * @param in The input stream containing the encoded bytes.
- * @return The decoded value.
- * @exception EOFException If the stream ends before a value has been decoded.
- */
- public static int decode(InputStream in) throws IOException {
- int value = 0;
- while (true) {
- int first = in.read();
- if (first < 0) {
- throw new EOFException();
- }
- value |= first & 0x7F;
- if ((first & 0x80) == 0) {
- return value;
- }
- value <<= 7;
- }
- }
-
- /**
- * The default ctor is made private because all methods of this class are static.
- */
- private Vint8() {
- // Just making it impossible to instantiate.
- }
-
-}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,39 +27,32 @@
* read more on the two implementations {@link FourFlagsIntEncoder} and
* {@link EightFlagsIntEncoder}.
*
- * IntEncoder fourFlags =
- * new SortingEncoderFilter(new UniqueValuesIntEncoder(new DGapIntEncoder(new FlagsIntEncoderImpl())));
- *
indicator, one flag (1-bits) at a time, and decodes extra
- * data using {@link VInt8IntDecoder}.
+ * Decodes values encoded with {@link EightFlagsIntEncoder}.
*
- * @see EightFlagsIntEncoder
* @lucene.experimental
*/
public class EightFlagsIntDecoder extends IntDecoder {
- /**
+ /*
* Holds all combinations of indicator for fast decoding (saves time
* on real-time bit manipulation)
*/
- private static final byte[][] decodeTable = new byte[256][8];
+ private static final byte[][] DECODE_TABLE = new byte[256][8];
/** Generating all combinations of indicator into separate flags. */
static {
@@ -42,48 +39,39 @@
--i;
for (int j = 8; j != 0;) {
--j;
- decodeTable[i][j] = (byte) ((i >>> j) & 0x1);
+ DECODE_TABLE[i][j] = (byte) ((i >>> j) & 0x1);
}
}
}
- private final IntDecoder decoder = new VInt8IntDecoder();
+ @Override
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ // read indicator
+ int indicator = buf.bytes[buf.offset++] & 0xFF;
+ int ordinal = 0;
- /** The indicator for decoding a chunk of 8 integers. */
- private int indicator;
+ int capacityNeeded = values.length + 8;
+ if (values.ints.length < capacityNeeded) {
+ values.grow(capacityNeeded);
+ }
- /** Used as an ordinal of 0 - 7, as the decoder decodes chunks of 8 integers. */
- private int ordinal = 0;
-
- @Override
- public long decode() throws IOException {
- // If we've decoded 8 integers, read the next indicator.
- if ((ordinal & 0x7) == 0) {
- indicator = in.read();
- if (indicator < 0) {
- return EOS;
+ // process indicator, until we read 8 values, or end-of-buffer
+ while (ordinal != 8) {
+ if (DECODE_TABLE[indicator][ordinal++] == 0) {
+ if (buf.offset == upto) { // end of buffer
+ return;
+ }
+ // decode the value from the stream.
+ values.ints[values.length++] = VInt8.decode(buf) + 2;
+ } else {
+ values.ints[values.length++] = 1;
+ }
}
- ordinal = 0;
}
-
- if (decodeTable[indicator][ordinal++] == 0) {
- // decode the value from the stream.
- long decode = decoder.decode();
- return decode == EOS ? EOS : decode + 2;
- }
-
- return 1;
}
@Override
- public void reInit(InputStream in) {
- super.reInit(in);
- decoder.reInit(in);
- ordinal = 0;
- indicator = 0;
- }
-
- @Override
public String toString() {
return "EightFlags (VInt8)";
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java (working copy)
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,14 +21,15 @@
*/
/**
- * A {@link ChunksIntEncoder} which encodes data in chunks of 8. Every group starts with a single
- * byte (called indicator) which represents 8 - 1 bit flags, where the value:
+ * A {@link ChunksIntEncoder} which encodes data in chunks of 8. Every group
+ * starts with a single byte (called indicator) which represents 8 - 1 bit
+ * flags, where the value:
*
*
* Encoding example:
*
- * Since value 0 is illegal, and 1 is encoded in the indicator, the actual
- * value that is encoded is value-2, which saves some more bits.
+ * Since value 0 is illegal, and 1 is encoded in the indicator, the actual value
+ * that is encoded is value-2, which saves some more bits.
*
@@ -46,28 +48,36 @@
*/
public class EightFlagsIntEncoder extends ChunksIntEncoder {
- /**
+ /*
* Holds all combinations of indicator flags for fast encoding (saves
* time on bit manipulation at encode time)
*/
- private static byte[] encodeTable = new byte[] { 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, (byte) 0x80 };
+ private static final byte[] ENCODE_TABLE = new byte[] { 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, (byte) 0x80 };
public EightFlagsIntEncoder() {
super(8);
}
@Override
- public void encode(int data) throws IOException {
- if (data == 1) {
- indicator |= encodeTable[ordinal];
- } else {
- encodeQueue[encodeQueueSize++] = data - 2;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ if (value == 1) {
+ indicator |= ENCODE_TABLE[ordinal];
+ } else {
+ encodeQueue.ints[encodeQueue.length++] = value - 2;
+ }
+ ++ordinal;
+
+ // encode the chunk and the indicator
+ if (ordinal == 8) {
+ encodeChunk(buf);
+ }
}
- ++ordinal;
-
- // If 8 values were encoded thus far, 'flush' them including the indicator.
- if ((ordinal & 0x7) == 0) {
- encodeChunk();
+
+ // encode remaining values
+ if (ordinal != 0) {
+ encodeChunk(buf);
}
}
@@ -78,7 +88,7 @@
@Override
public String toString() {
- return "EightFlags (" + encoder.toString() + ")";
+ return "EightFlags (VInt)";
}
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,11 +21,8 @@
*/
/**
- * Decodes data which was encoded by {@link FourFlagsIntEncoder}. Scans
- * the
indicator, one flag (1-bits) at a time, and decodes extra
- * data using {@link VInt8IntDecoder}.
+ * Decodes values encoded with {@link FourFlagsIntEncoder}.
*
- * @see FourFlagsIntEncoder
* @lucene.experimental
*/
public class FourFlagsIntDecoder extends IntDecoder {
@@ -34,7 +31,7 @@
* Holds all combinations of indicator for fast decoding (saves time
* on real-time bit manipulation)
*/
- private final static byte[][] decodeTable = new byte[256][4];
+ private final static byte[][] DECODE_TABLE = new byte[256][4];
/** Generating all combinations of indicator into separate flags. */
static {
@@ -42,49 +39,39 @@
--i;
for (int j = 4; j != 0;) {
--j;
- decodeTable[i][j] = (byte) ((i >>> (j << 1)) & 0x3);
+ DECODE_TABLE[i][j] = (byte) ((i >>> (j << 1)) & 0x3);
}
}
}
- private final IntDecoder decoder = new VInt8IntDecoder();
-
- /** The indicator for decoding a chunk of 4 integers. */
- private int indicator;
-
- /** Used as an ordinal of 0 - 3, as the decoder decodes chunks of 4 integers. */
- private int ordinal = 0;
-
@Override
- public long decode() throws IOException {
- // If we've decoded 8 integers, read the next indicator.
- if ((ordinal & 0x3) == 0) {
- indicator = in.read();
- if (indicator < 0) {
- return EOS;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ // read indicator
+ int indicator = buf.bytes[buf.offset++] & 0xFF;
+ int ordinal = 0;
+
+ int capacityNeeded = values.length + 4;
+ if (values.ints.length < capacityNeeded) {
+ values.grow(capacityNeeded);
}
- ordinal = 0;
+
+ while (ordinal != 4) {
+ byte decodeVal = DECODE_TABLE[indicator][ordinal++];
+ if (decodeVal == 0) {
+ if (buf.offset == upto) { // end of buffer
+ return;
+ }
+ // decode the value from the stream.
+ values.ints[values.length++] = VInt8.decode(buf) + 4;
+ } else {
+ values.ints[values.length++] = decodeVal;
+ }
+ }
}
-
- byte decodeVal = decodeTable[indicator][ordinal++];
- if (decodeVal == 0) {
- // decode the value from the stream.
- long decode = decoder.decode();
- return decode == EOS ? EOS : decode + 4;
- }
-
- return decodeVal;
}
@Override
- public void reInit(InputStream in) {
- super.reInit(in);
- decoder.reInit(in);
- ordinal = 0;
- indicator = 0;
- }
-
- @Override
public String toString() {
return "FourFlags (VInt8)";
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java (working copy)
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -48,11 +49,11 @@
*/
public class FourFlagsIntEncoder extends ChunksIntEncoder {
- /**
+ /*
* Holds all combinations of indicator flags for fast encoding (saves
* time on bit manipulation @ encode time)
*/
- private static byte[][] encodeTable = new byte[][] {
+ private static final byte[][] ENCODE_TABLE = new byte[][] {
new byte[] { 0x00, 0x00, 0x00, 0x00 },
new byte[] { 0x01, 0x04, 0x10, 0x40 },
new byte[] { 0x02, 0x08, 0x20, (byte) 0x80 },
@@ -63,26 +64,26 @@
super(4);
}
- /**
- * Small values (<=3) are stored in the indicator while larger
- * values are saved for later encoding in the {@link #encodeQueue}. Since
- * Vint8 will only encode values larger or equal to 4, the values saves for
- * encoded are transformed to (value - 4).
- * When a chunk is ready (got 4 values), the {@link #encodeChunk()}
- * takes control.
- */
@Override
- public void encode(int data) throws IOException {
- if (data <= 3) {
- indicator |= encodeTable[data][ordinal];
- } else {
- encodeQueue[encodeQueueSize++] = data - 4;
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ if (value <= 3) {
+ indicator |= ENCODE_TABLE[value][ordinal];
+ } else {
+ encodeQueue.ints[encodeQueue.length++] = value - 4;
+ }
+ ++ordinal;
+
+ // encode the chunk and the indicator
+ if (ordinal == 4) {
+ encodeChunk(buf);
+ }
}
- ++ordinal;
-
- // If 4 values were encoded thus far, 'flush' them including the indicator.
- if ((ordinal & 0x3) == 0) {
- encodeChunk();
+
+ // encode remaining values
+ if (ordinal != 0) {
+ encodeChunk(buf);
}
}
@@ -93,7 +94,7 @@
@Override
public String toString() {
- return "FourFlags (" + encoder.toString() + ")";
+ return "FourFlags (VInt)";
}
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,33 +21,50 @@
*/
/**
- * Decodes integers from a set {@link InputStream}. For re-usability, the
- * decoder's input stream can be set by ({@link #reInit(InputStream)}).
- * By design, Decoders are NOT thread-safe.
+ * Decodes integers from a set {@link BytesRef}.
*
* @lucene.experimental
*/
public abstract class IntDecoder {
- /** A special long value which is used to indicate end-of-stream has reached. */
- public static final long EOS = 0x100000000L;
-
- /** Input stream from which the encoded bytes are read */
- protected InputStream in;
-
- /** Sets the input stream from which the encoded data is read. */
- public void reInit(InputStream in) {
- this.in = in;
+ /**
+ * Performs the actual decoding. Values should be read from
+ * {@link BytesRef#offset} up to {@code upto}. Also, {@code values} offset and
+ * length are set to 0 and the encoder is expected to update
+ * {@link IntsRef#length}, but not {@link IntsRef#offset}.
+ *
+ *
- * Once this method was called, no further calls to {@link #encode(int)
- * encode} should be made before first calling {@link #reInit(OutputStream)
- * reInit}.
- * super.close() or close the output stream themselves.
+ * Called before {@link #doEncode(IntsRef, BytesRef, int)} so that encoders
+ * can reset their state.
*/
- @Override
- public void close() throws IOException {
- if (out != null) {
- out.close();
- }
+ protected void reset() {
+ // do nothing by default
}
/**
- * Encodes an integer to the output stream given in
- * {@link #reInit(OutputStream) reInit}
+ * Encodes the values to the given buffer. Note that the buffer's offset and
+ * length are set to 0.
*/
- public abstract void encode(int value) throws IOException;
+ public final void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
+ reset();
+ doEncode(values, buf, values.offset + values.length);
+ assert buf.offset == 0 : "offset should not have been modified by the encoder.";
+ }
- /**
- * Returns an {@link IntDecoder} which matches this encoder. Every encoder
- * must return an {@link IntDecoder} and null is not a valid
- * value. If an encoder is just a filter, it should at least return its
- * wrapped encoder's matching decoder.
- *
- * new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEnoder()));
- * {@link UniqueValuesIntEncoder} followed by {@link DGapIntEncoder}
-
+ *
+ * Examples:
+ *
+ *
+ * {@link #encode(int, BytesRef)} and {@link #decode(BytesRef)} will correctly
+ * handle any 32-bit integer, but for negative numbers, and positive numbers
+ * with more than 28 significant bits, encoding requires 5 bytes; this is not an
+ * efficient encoding scheme for large positive numbers or any negative number.
+ *
+ * @lucene.experimental
+ */
+public class VInt8 {
+
+ /** The maximum number of bytes needed to encode an integer. */
+ public static final int MAXIMUM_BYTES_NEEDED = 5;
+
+ /** Returns the number of bytes needed to encode {@code number}. */
+ public static int bytesNeeded(int number) {
+ if ((number & ~0x7F) == 0) {
+ return 1;
+ } else if ((number & ~0x3FFF) == 0) {
+ return 2;
+ } else if ((number & ~0x1FFFFF) == 0) {
+ return 3;
+ } else if ((number & ~0xFFFFFFF) == 0) {
+ return 4;
+ } else {
+ return 5;
+ }
+ }
+
+ /**
+ * Decodes an int from the given bytes, starting at {@link BytesRef#offset}.
+ * Returns the decoded bytes and updates {@link BytesRef#offset}.
+ */
+ public static int decode(BytesRef bytes) {
+ /*
+ This is the original code of this method, but a Hotspot bug
+ corrupted the for-loop of DataInput.readVInt() (see LUCENE-2975)
+ so the loop was unwounded here too, to be on the safe side
+ int value = 0;
+ while (true) {
+ byte first = bytes.bytes[bytes.offset++];
+ value |= first & 0x7F;
+ if ((first & 0x80) == 0) {
+ return value;
+ }
+ value <<= 7;
+ }
+ */
+
+ // byte 1
+ byte b = bytes.bytes[bytes.offset++];
+ if (b >= 0) return b;
+
+ // byte 2
+ int value = b & 0x7F;
+ b = bytes.bytes[bytes.offset++];
+ value = (value << 7) | b & 0x7F;
+ if (b >= 0) return value;
+
+ // byte 3
+ b = bytes.bytes[bytes.offset++];
+ value = (value << 7) | b & 0x7F;
+ if (b >= 0) return value;
+
+ // byte 4
+ b = bytes.bytes[bytes.offset++];
+ value = (value << 7) | b & 0x7F;
+ if (b >= 0) return value;
+
+ // byte 5
+ b = bytes.bytes[bytes.offset++];
+ return (value << 7) | b & 0x7F;
+ }
+
+ /**
+ * Encodes the given number into bytes, starting at {@link BytesRef#length}.
+ * Assumes that the array is large enough.
+ */
+ public static void encode(int value, BytesRef bytes) {
+ if ((value & ~0x7F) == 0) {
+ bytes.bytes[bytes.length] = (byte) value;
+ bytes.length++;
+ } else if ((value & ~0x3FFF) == 0) {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 1] = (byte) (value & 0x7F);
+ bytes.length += 2;
+ } else if ((value & ~0x1FFFFF) == 0) {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ bytes.bytes[bytes.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 2] = (byte) (value & 0x7F);
+ bytes.length += 3;
+ } else if ((value & ~0xFFFFFFF) == 0) {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ bytes.bytes[bytes.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ bytes.bytes[bytes.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 3] = (byte) (value & 0x7F);
+ bytes.length += 4;
+ } else {
+ bytes.bytes[bytes.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+ bytes.bytes[bytes.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ bytes.bytes[bytes.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ bytes.bytes[bytes.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ bytes.bytes[bytes.length + 4] = (byte) (value & 0x7F);
+ bytes.length += 5;
+ }
+ }
+
+ private VInt8() {
+ // Just making it impossible to instantiate.
+ }
+
+}
Property changes on: lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java (working copy)
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,33 +21,19 @@
*/
/**
- * An {@link IntDecoder} which can decode values encoded by
- * {@link VInt8IntEncoder}.
+ * Decodes values encoded by {@link VInt8IntEncoder}.
*
* @lucene.experimental
*/
public class VInt8IntDecoder extends IntDecoder {
- private boolean legalEOS = true;
-
@Override
- public long decode() throws IOException {
- int value = 0;
- while (true) {
- int first = in.read();
- if (first < 0) {
- if (!legalEOS) {
- throw new IOException("Unexpected End-Of-Stream");
- }
- return EOS;
+ protected void doDecode(BytesRef buf, IntsRef values, int upto) {
+ while (buf.offset < upto) {
+ if (values.length == values.ints.length) {
+ values.grow(values.length + 10); // grow by few items, however not too many
}
- value |= first & 0x7F;
- if ((first & 0x80) == 0) {
- legalEOS = true;
- return value;
- }
- legalEOS = false;
- value <<= 7;
+ values.ints[values.length++] = VInt8.decode(buf);
}
}
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java (working copy)
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -49,28 +50,15 @@
public class VInt8IntEncoder extends IntEncoder {
@Override
- public void encode(int value) throws IOException {
- if ((value & ~0x7F) == 0) {
- out.write(value);
- } else if ((value & ~0x3FFF) == 0) {
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else if ((value & ~0x1FFFFF) == 0) {
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else if ((value & ~0xFFFFFFF) == 0) {
- out.write(0x80 | (value >> 21));
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else {
- out.write(0x80 | (value >> 28));
- out.write(0x80 | (value >> 21));
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
+ protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ int maxBytesNeeded = 5 * values.length; // at most 5 bytes per VInt
+ if (buf.bytes.length < maxBytesNeeded) {
+ buf.grow(maxBytesNeeded);
}
+
+ for (int i = values.offset; i < upto; i++) {
+ VInt8.encode(values.ints[i], buf);
+ }
}
@Override
Index: lucene/facet/src/java/org/apache/lucene/util/encoding/package.html
===================================================================
--- lucene/facet/src/java/org/apache/lucene/util/encoding/package.html (revision 1431562)
+++ lucene/facet/src/java/org/apache/lucene/util/encoding/package.html (working copy)
@@ -25,49 +25,8 @@
encoders there is a matching {@link
org.apache.lucene.util.encoding.IntDecoder} implementation (not all
encoders need a decoder).
-
-Each encoder also implements {@link
-org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} which returns the matching decoder for this encoder.
-As mentioned above, not all encoders have a matching decoder (like some
-encoder filters which are explained next), however every encoder should
-return a decoder following a call to that method. To complete the
-example above, one can easily iterate over the decoded values like this:
-
-int[] data = <the values to encode>
-IntEncoder encoder = new VInt8IntEncoder();
-OutputStream out = new ByteArrayOutputStream();
-encoder.reInit(out);
-for (int val : data) {
- encoder.encode(val);
-}
-encoder.close();
-
-// Print the bytes in binary
-byte[] bytes = out.toByteArray();
-for (byte b : bytes) {
- System.out.println(Integer.toBinaryString(b));
-}
-
-
-IntDecoder d = e.createMatchingDecoder();
-d.reInit(new ByteArrayInputStream(bytes));
-long val;
-while ((val = d.decode()) != IntDecoder.EOS) {
- System.out.println(val);
-}
-Extending IntEncoder
-Extending {@link org.apache.lucene.util.encoding.IntEncoder} is a very
-easy task. One only needs to implement {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} and
-{@link org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} as the base implementation takes care of
-re-initializing the output stream and closing it. The following example
-illustrates how can one write an encoder (and a matching decoder) which
-'tags' the stream with type/ID of the encoder. Such tagging is important
-in scenarios where an application uses different encoders for different
-streams, and wants to manage some sort of mapping between an encoder ID
-to an IntEncoder/Decoder implementation, so a proper decoder will be
-initialized on the fly:
-
-And the matching decoder:
-
-public class TaggingIntEncoder extends IntEncoderFilter {
-
- public TaggingIntEncoder(IntEncoder encoder) {
- super(encoder);
- }
-
- @Override
- public void encode(int value) throws IOException {
- encoder.encode(value);
- }
-
- @Override
- public IntDecoder createMatchingDecoder() {
- return new TaggingIntDecoder();
- }
-
- @Override
- public void reInit(OutputStream out) {
- super.reInit(os);
- // Assumes the application has a static EncodersMap class which is able to
- // return a unique ID for a given encoder.
- int encoderID = EncodersMap.getID(encoder);
- this.out.write(encoderID);
- }
-
- @Override
- public String toString() {
- return "Tagging (" + encoder.toString() + ")";
- }
-
-}
-
-The example implements
-public class TaggingIntDecoder extends IntDecoder {
-
- // Will be initialized upon calling reInit.
- private IntDecoder decoder;
-
- @Override
- public void reInit(InputStream in) {
- super.reInit(in);
-
- // Read the ID of the encoder that tagged this stream.
- int encoderID = in.read();
-
- // Assumes EncodersMap can return the proper IntEncoder given the ID.
- decoder = EncodersMap.getEncoder(encoderID).createMatchingDecoder();
- }
-
- @Override
- public long decode() throws IOException {
- return decoder.decode();
- }
-
- @Override
- public String toString() {
- return "Tagging (" + decoder == null ? "none" : decoder.toString() + ")";
- }
-
-}
-TaggingIntEncoder as a filter over another
-encoder. Even though it does not do any filtering on the actual values, it feels
-right to present it as a filter. Anyway, this is just an example code and one
-can choose to implement it however it makes sense to the application. For
-simplicity, error checking was omitted from the sample code.
+values in ascending order before they are sent for encoding.