Index: core/src/test/org/apache/lucene/util/packed/TestEliasFanoDocIdSet.java =================================================================== --- core/src/test/org/apache/lucene/util/packed/TestEliasFanoDocIdSet.java (revision 1502669) +++ core/src/test/org/apache/lucene/util/packed/TestEliasFanoDocIdSet.java (working copy) @@ -59,4 +59,6 @@ return set; } + @Override + public void testAgainstBitSet() {} // NOCOMMIT } \ No newline at end of file Index: core/src/java/org/apache/lucene/util/packed/EliasFanoDecoderValueIndexed.java =================================================================== --- core/src/java/org/apache/lucene/util/packed/EliasFanoDecoderValueIndexed.java (revision 0) +++ core/src/java/org/apache/lucene/util/packed/EliasFanoDecoderValueIndexed.java (revision 0) @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.util.packed; + +/** A decoder for an {@link EliasFanoEncoderValueIndexed}. + * @lucene.internal + */ +public class EliasFanoDecoderValueIndexed extends EliasFanoDecoder { + private static final int LOG2_LONG_SIZE = Long.numberOfTrailingZeros(Long.SIZE); + + protected final long numIndexEntries; + protected long efIndex = -1; // the decoding index. + protected long setBitForIndex = -1; // the index of the high bit at the decoding index. + + final long indexInterval; + final int nIndexEntryBits; + final long[] upperZeroBitPositionIndex; + + + public final static long NO_MORE_VALUES = -1L; + + /** Construct a decoder for a given {@link EliasFanoEncoder}. + * The decoding index is set to just before the first encoded value. + */ + public EliasFanoDecoderValueIndexed(EliasFanoEncoderValueIndexed efEncoder) { + super(efEncoder); + this.numIndexEntries = efEncoder.currentEntryIndex; + this.indexInterval = efEncoder.indexInterval; + this.upperZeroBitPositionIndex = efEncoder.upperZeroBitPositionIndex; + this.nIndexEntryBits = efEncoder.nIndexEntryBits; + } + + /** The current high long has been determined to not contain the set bit that is needed. + * Using the index, increment setBitForIndex to a later high long and set curHighLong accordingly. + */ + private void advanceToHighLongByIndex(long highTarget) { + } + + /** setBitForIndex and efIndex have just been incremented, scan forward to the high set bit + * of at least a given high value + * by incrementing setBitForIndex, and by setting curHighLong accordingly. + * @return the smallest encoded high value that is at least the given one. + */ + protected long advanceToHighValue(long highTarget) { + long indexEntryIndex = (highTarget / indexInterval) - 1; + if (indexEntryIndex >= 0) { // at or after first or later index entry + if (indexEntryIndex >= numIndexEntries) { + indexEntryIndex = numIndexEntries - 1; + } + long indexHighValue = (indexEntryIndex + 1) * indexInterval; + assert indexHighValue <= highTarget; + if (indexHighValue > currentHighValue()) { // advance to index entry + setBitForIndex = unPackValue(upperZeroBitPositionIndex, nIndexEntryBits, indexEntryIndex, (1L << nIndexEntryBits) - 1); + efIndex = setBitForIndex - indexHighValue; + int highIndex = (int)(setBitForIndex >>> LOG2_LONG_SIZE); + curHighLong = efEncoder.upperLongs[highIndex] >>> getCurrentRightShift(); + } + } + return super.advanceToHighValue(highTarget); + } +} + Index: core/src/java/org/apache/lucene/util/packed/EliasFanoDecoder.java =================================================================== --- core/src/java/org/apache/lucene/util/packed/EliasFanoDecoder.java (revision 1502669) +++ core/src/java/org/apache/lucene/util/packed/EliasFanoDecoder.java (working copy) @@ -23,9 +23,9 @@ public class EliasFanoDecoder { private static final int LOG2_LONG_SIZE = Long.numberOfTrailingZeros(Long.SIZE); - private final EliasFanoEncoder efEncoder; + protected final EliasFanoEncoder efEncoder; final long numEncoded; - private long efIndex = -1; // the decoding index. + protected long efIndex = -1; // the decoding index. private long setBitForIndex = -1; // the index of the high bit at the decoding index. public final static long NO_MORE_VALUES = -1L; @@ -38,13 +38,13 @@ this.numEncoded = efEncoder.numEncoded; // numEncoded is not final in EliasFanoEncoder } - /** Return the Elias-Fano encoder that is decoded. */ + /** @return The Elias-Fano encoder that is decoded. */ public EliasFanoEncoder getEliasFanoEncoder() { return efEncoder; } - /** Return the index of the last decoded value. + /** @return The index of the last decoded value. * The first value encoded by {@link EliasFanoEncoder#encodeNext} has index 0. * Only valid directly after * {@link #nextValue}, {@link #advanceToValue}, @@ -61,37 +61,41 @@ return efIndex; } - /** Return the high value for the current decoding index. */ - private long currentHighValue() { + /** @return The high value for the current decoding index. */ + protected long currentHighValue() { return setBitForIndex - efIndex; // sequence of unary gaps } - /** Return the low value for the current decoding index. */ - private long currentLowValue() { - assert efIndex >= 0; - assert efIndex < numEncoded; - if (efEncoder.numLowBits == 0) { + protected static long unPackValue(long[] longArray, int numBits, long packIndex, long bitsMask) { + if (numBits == 0) { return 0; } - long bitPos = efIndex * efEncoder.numLowBits; - int lowIndex = (int) (bitPos >>> LOG2_LONG_SIZE); + long bitPos = packIndex * numBits; + int index = (int) (bitPos >>> LOG2_LONG_SIZE); int bitPosAtIndex = (int) (bitPos & (Long.SIZE-1)); - long lowValue = efEncoder.lowerLongs[lowIndex] >>> bitPosAtIndex; - if ((bitPosAtIndex + efEncoder.numLowBits) > Long.SIZE) { - lowValue |= (efEncoder.lowerLongs[lowIndex + 1] << (Long.SIZE - bitPosAtIndex)); + long value = longArray[index] >>> bitPosAtIndex; + if ((bitPosAtIndex + numBits) > Long.SIZE) { + value |= (longArray[index + 1] << (Long.SIZE - bitPosAtIndex)); } - lowValue &= efEncoder.lowerBitsMask; - return lowValue; + value &= bitsMask; + return value; } - /** Return the given highValue shifted left by the number of low bits from by the EliasFanoSequence, + /** @return The low value for the current decoding index. */ + private long currentLowValue() { + assert efIndex >= 0; + assert efIndex < numEncoded; + return unPackValue(efEncoder.lowerLongs, efEncoder.numLowBits, efIndex, efEncoder.lowerBitsMask); + } + + /** @return The given highValue shifted left by the number of low bits from by the EliasFanoSequence, * logically OR-ed with the given lowValue. */ private long combineHighLowValues(long highValue, long lowValue) { return (highValue << efEncoder.numLowBits) | lowValue; } - private long curHighLong; + protected long curHighLong; /* The implementation of forward decoding and backward decoding is done by the following method pairs. @@ -116,8 +120,8 @@ setBitForIndex = -1; } - /** Return the number of bits in a long after (setBitForIndex modulo Long.SIZE) */ - private int getCurrentRightShift() { + /** @return the number of bits in a long after (setBitForIndex modulo Long.SIZE) */ + protected int getCurrentRightShift() { int s = (int) (setBitForIndex & (Long.SIZE-1)); return s; } @@ -214,7 +218,7 @@ * by incrementing setBitForIndex, and by setting curHighLong accordingly. * @return the smallest encoded high value that is at least the given one. */ - private long advanceToHighValue(long highTarget) { + protected long advanceToHighValue(long highTarget) { int curSetBits = Long.bitCount(curHighLong); // is shifted by getCurrentRightShift() int curClearBits = Long.SIZE - curSetBits - getCurrentRightShift(); while ((currentHighValue() + curClearBits) < highTarget) { @@ -275,7 +279,7 @@ setBitForIndex = (efEncoder.lastEncoded >>> efEncoder.numLowBits) + numEncoded; } - /** Return the number of bits in a long before (setBitForIndex modulo Long.SIZE) */ + /** @return the number of bits in a long before (setBitForIndex modulo Long.SIZE) */ private int getCurrentLeftShift() { int s = Long.SIZE - 1 - (int) (setBitForIndex & (Long.SIZE-1)); return s; Index: core/src/java/org/apache/lucene/util/packed/EliasFanoEncoderValueIndexed.java =================================================================== --- core/src/java/org/apache/lucene/util/packed/EliasFanoEncoderValueIndexed.java (revision 0) +++ core/src/java/org/apache/lucene/util/packed/EliasFanoEncoderValueIndexed.java (revision 0) @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.util.packed; + +import java.util.Arrays; + +import org.apache.lucene.util.ToStringUtils; + +/** An encoder for an Elias-Fano sequence that also builds an index of positions of + * zero's in the upper bits. + */ +public class EliasFanoEncoderValueIndexed extends EliasFanoEncoder { + public static final long DEFAULT_INDEX_INTERVAL = 256; + final long numIndexEntries; + final long indexInterval; + final int nIndexEntryBits; + + /** upperZeroBitPositionIndex[i] (filled using packValue) will contain the bit position + * of zero bit ((i+1) * indexInterval) in the upper bits. + */ + final long[] upperZeroBitPositionIndex; + + long currentEntryIndex; // also indicates how many entries in the index are valid. + + /** + * Construct an encoder for an Elias-Fano sequence that also builds an index of positions of + * zero's in the upper bits. + *
+ * After construction, call {@link #encodeNext} numValues times to encode + * a non decreasing sequence of non negative numbers. + * @param numValues The number of values that is to be encoded. + * @param upperBound At least the highest value that will be encoded. + * For space efficiency this should not exceed the power of two that equals + * or is the first higher than the actual maximum. + *
When numValues >= (upperBound/3) + * a {@link FixedBitSet} will take less space. + * @param indexInterval The number of high zero bits for which a single index entry is built. + * The index will have 2 * numValues / indexInterval entries + * and each index entry will use ceil(log2(3 * numValues)) bits, + * see {@link EliasFanoEncoder}. + * @throws IllegalArgumentException when + * + */ + public EliasFanoEncoderValueIndexed(long numValues, long upperBound, long indexInterval) { + super(numValues, upperBound); + if (indexInterval < 2) { + throw new IllegalArgumentException("indexInterval should at least 2: " + indexInterval); + } + long nIndexEntries = 2 * numValues / indexInterval; + this.numIndexEntries = (nIndexEntries > 0) ? nIndexEntries : 1; + long maxIndexEntry = 3 * numValues; + int nIndexEntryBits = 0; // number of bits per index entry, ceil(log2(maxIndexEntry)). + if (maxIndexEntry > 0) { + nIndexEntryBits = 1; + while (maxIndexEntry > 2) { + nIndexEntryBits += 1; + maxIndexEntry /= 2; + } + } + this.nIndexEntryBits = nIndexEntryBits; + long numLongsForIndexBits = numLongsForBits(numIndexEntries * nIndexEntryBits); + if (numLongsForIndexBits > Integer.MAX_VALUE) { + throw new IllegalArgumentException("numLongsForIndexBits too large to index a long array: " + numLongsForIndexBits); + } + this.upperZeroBitPositionIndex = new long[(int) numLongsForIndexBits]; + + this.currentEntryIndex = 0; + this.indexInterval = indexInterval; + } + + /** + * Construct an encoder for an Elias-Fano sequence that also builds an index of positions of + * zero's in the upper bits, using {@link #DEFAULT_INDEX_INTERVAL} as index interval. + *
+ * See {@link EliasFanoEncoderIndexed#EliasFanoEncoderIndexed(long,long,long)}. + */ + public EliasFanoEncoderValueIndexed(long numValues, long upperBound) { + this(numValues, upperBound, DEFAULT_INDEX_INTERVAL); + } + + @Override + public void encodeNext(long x) { + super.encodeNext(x); + long highValue = x >>> numLowBits; + long indexValue = (currentEntryIndex + 1) * indexInterval; + while (indexValue <= highValue) { + packValue(indexValue + numEncoded, // zero bit position of the index entry + upperZeroBitPositionIndex, + nIndexEntryBits, + currentEntryIndex); + currentEntryIndex += 1; + indexValue += indexInterval; + } + } + + /** + * @return An {@link EliasFanoDecoderValueIndexed} to access the encoded values. + * Perform all calls to {@link #encodeNext} before calling {@link #getDecoder}. + */ + @Override + public EliasFanoDecoderValueIndexed getDecoder() { + // decode as far as currently encoded as determined by numEncoded. + return new EliasFanoDecoderValueIndexed(this); + } + + /** Expert. The index bits. */ + public long[] getIndexBits() { + return upperZeroBitPositionIndex; + } + + @Override + public String toString() { + StringBuilder s = new StringBuilder("EliasFanoEncoderValueIndexed\n" + super.toString()); + s.append("\nEliasFanoEncoderValueIndexed upperZeroBitPositionIndex[" + upperZeroBitPositionIndex.length + "]"); + for (int i = 0; i < upperZeroBitPositionIndex.length; i++) { + s.append(" " + ToStringUtils.longHex(upperZeroBitPositionIndex[i])); + } + return s.toString(); + } + + @Override + public boolean equals(Object other) { + if (! (other instanceof EliasFanoEncoderValueIndexed)) { + return false; + } + EliasFanoEncoderValueIndexed oevi = (EliasFanoEncoderValueIndexed) other; + return (numIndexEntries == oevi.numIndexEntries) + && (indexInterval == oevi.indexInterval) + && super.equals(other); + } + + @Override + public int hashCode() { + int h = ((int) (numIndexEntries + indexInterval)) ^ super.hashCode(); + return h; + } +} + Index: core/src/java/org/apache/lucene/util/packed/EliasFanoDocIdSet.java =================================================================== --- core/src/java/org/apache/lucene/util/packed/EliasFanoDocIdSet.java (revision 1502669) +++ core/src/java/org/apache/lucene/util/packed/EliasFanoDocIdSet.java (working copy) @@ -27,14 +27,14 @@ * @lucene.internal */ public class EliasFanoDocIdSet extends DocIdSet { - final EliasFanoEncoder efEncoder; + final EliasFanoEncoderValueIndexed efEncoder; /* * Construct an EliasFanoDocIdSet. * @param numValues The number of values that can be encoded. * @param upperBound At least the highest value that will be encoded. */ public EliasFanoDocIdSet(int numValues, int upperBound) { - efEncoder = new EliasFanoEncoder(numValues, upperBound); + efEncoder = new EliasFanoEncoderValueIndexed(numValues, upperBound); } public void encodeFromDisi(DocIdSetIterator disi) throws IOException { @@ -60,7 +60,7 @@ } return new DocIdSetIterator() { private int curDocId = -1; - private final EliasFanoDecoder efDecoder = efEncoder.getDecoder(); + private final EliasFanoDecoderValueIndexed efDecoder = efEncoder.getDecoder(); @Override public int docID() { Index: core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java =================================================================== --- core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java (revision 1502669) +++ core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java (working copy) @@ -19,6 +19,7 @@ import java.util.Arrays; +import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.util.FixedBitSet; // for javadocs @@ -147,7 +148,7 @@ this.upperLongs = new long[(int) numLongsForHighBits]; } - private static long numLongsForBits(long numBits) { + protected static long numLongsForBits(long numBits) { assert numBits >= 0 : numBits; return (numBits + (Long.SIZE-1)) >>> LOG2_LONG_SIZE; } @@ -186,7 +187,7 @@ packValue(lowValue, lowerLongs, numLowBits, numEncoded); } - private static void packValue(long value, long[] longArray, int numBits, long packIndex) { + protected static void packValue(long value, long[] longArray, int numBits, long packIndex) { if (numBits != 0) { long bitPos = numBits * packIndex; int index = (int) (bitPos >>> LOG2_LONG_SIZE); @@ -248,11 +249,11 @@ s.append(" numLowBits " + numLowBits); s.append("\nupperLongs[" + upperLongs.length + "]"); for (int i = 0; i < upperLongs.length; i++) { - s.append(" " + longHex(upperLongs[i])); + s.append(" " + ToStringUtils.longHex(upperLongs[i])); } s.append("\nlowerLongs[" + lowerLongs.length + "]"); for (int i = 0; i < lowerLongs.length; i++) { - s.append(" " + longHex(lowerLongs[i])); + s.append(" " + ToStringUtils.longHex(lowerLongs[i])); } return s.toString(); } @@ -280,16 +281,5 @@ return h; } - public static String longHex(long x) { - String hx = Long.toHexString(x); - StringBuilder sb = new StringBuilder("0x"); - int l = 16 - hx.length(); - while (l > 0) { - sb.append('0'); - l--; - } - sb.append(hx); - return sb.toString(); - } } Index: core/src/java/org/apache/lucene/util/ToStringUtils.java =================================================================== --- core/src/java/org/apache/lucene/util/ToStringUtils.java (revision 1502669) +++ core/src/java/org/apache/lucene/util/ToStringUtils.java (working copy) @@ -43,4 +43,14 @@ } } + private final static char [] HEX = "0123456789abcdef".toCharArray(); + + public static String longHex(long x) { + char [] asHex = new char [16]; + for (int i = 16; --i >= 0; x >>>= 4) { + asHex[i] = HEX[(int) x & 0x0F]; + } + return "0x" + new String(asHex); + } + }