Index: src/java/org/apache/lucene/util/DefaultMatcher.java
===================================================================
--- src/java/org/apache/lucene/util/DefaultMatcher.java (revision 0)
+++ src/java/org/apache/lucene/util/DefaultMatcher.java (revision 0)
@@ -0,0 +1,79 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.BitSet;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Matcher;
+
+/** Provides default Matchers for filtering. */
+public class DefaultMatcher {
+ /**
+ * Create a Matcher from a BitSet.
+ */
+ public static Matcher defaultMatcher(BitSet bits) {
+ return new BitSetMatcher(bits);
+ }
+
+ /**
+ * Create a Matcher from an OpenBitSet.
+ */
+ public static Matcher defaultMatcher(OpenBitSet bits) {
+ return new OpenBitSetMatcher(bits);
+ }
+
+
+ /** Prefer SortedVIntList over (Open)BitSet when it is smaller.
+ * This preference may have a performance penalty when
+ * skipTo() is used a lot on the Matcher provided by SortedVIntList.
+ * When that happens, a skip list implementation of SortedVIntList should
+ * be considered.
+ */
+ private static boolean preferVIntsOverBits(long cardinality, long length) {
+ return (cardinality < (length / SortedVIntList.BITS2VINTLIST_SIZE));
+ }
+
+ /** To be placed in a cache for a MatchFilter.
+ * Wraps a given BitSet or a SortedVIntList
+ * derived from the given BitSet when the SortedVIntList is smaller.
+ */
+ public static MatcherProvider makeMatcherProvider(final BitSet bits) {
+ if (preferVIntsOverBits(bits.cardinality(), bits.length())) {
+ return new SortedVIntList(bits);
+ } else {
+ return new MatcherProvider() { // wrap the BitSet
+ public Matcher getMatcher() { return new BitSetMatcher(bits); }
+ };
+ }
+ }
+
+ /** To be placed in a cache for a MatchFilter.
+ * Wraps a given OpenBitSet or a SortedVIntList
+ * derived from the given OpenBitSet when the SortedVIntList is smaller.
+ */
+ public static MatcherProvider makeMatcherProvider(final OpenBitSet bits) {
+ if (preferVIntsOverBits(bits.cardinality(), bits.size())) {
+ return new SortedVIntList(bits);
+ } else {
+ return new MatcherProvider() { // wrap the OpenBitSet
+ public Matcher getMatcher() { return new OpenBitSetMatcher(bits); }
+ };
+ }
+ }
+}
+
Property changes on: src/java/org/apache/lucene/util/DefaultMatcher.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/java/org/apache/lucene/util/MatcherProvider.java
===================================================================
--- src/java/org/apache/lucene/util/MatcherProvider.java (revision 0)
+++ src/java/org/apache/lucene/util/MatcherProvider.java (revision 0)
@@ -0,0 +1,26 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.search.Matcher;
+
+/** To be used in a cache to implement caching for a MatchFilter. */
+public interface MatcherProvider {
+ public Matcher getMatcher();
+}
+
Property changes on: src/java/org/apache/lucene/util/MatcherProvider.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/java/org/apache/lucene/util/BitSetMatcher.java
===================================================================
--- src/java/org/apache/lucene/util/BitSetMatcher.java (revision 0)
+++ src/java/org/apache/lucene/util/BitSetMatcher.java (revision 0)
@@ -0,0 +1,57 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.BitSet;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Matcher;
+
+/** A Matcher constructed from a BitSet */
+public class BitSetMatcher extends Matcher {
+ private BitSet bitset;
+ private int docNr = -1;
+
+ public BitSetMatcher(BitSet bitset) {
+ this.bitset = bitset;
+ }
+
+ public int doc() {
+ assert docNr != -1;
+ return docNr;
+ }
+
+ public boolean next() {
+ // (docNr + 1) on next line requires -1 initial value for docNr:
+ return checkNextDocNr(bitset.nextSetBit(docNr + 1));
+ }
+
+ public boolean skipTo(int skipDocNr) {
+ return checkNextDocNr( bitset.nextSetBit(skipDocNr));
+ }
+
+ private boolean checkNextDocNr(int d) {
+ if (d == -1) { // -1 returned by BitSet.nextSetBit() when exhausted
+ docNr = Integer.MAX_VALUE;
+ return false;
+ } else {
+ docNr = d;
+ return true;
+ }
+ }
+}
+
Property changes on: src/java/org/apache/lucene/util/BitSetMatcher.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/java/org/apache/lucene/util/OpenBitSet.java
===================================================================
--- src/java/org/apache/lucene/util/OpenBitSet.java (revision 0)
+++ src/java/org/apache/lucene/util/OpenBitSet.java (revision 0)
@@ -0,0 +1,766 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util; // copied from org.apache.solr.util, rev. 555343
+
+import java.util.Arrays;
+import java.io.Serializable;
+
+/** An "open" BitSet implementation that allows direct access to the array of words
+ * storing the bits.
+ *
+ * Unlike java.util.BitSet, the fact that bits are packed into an array of longs
+ * is part of the interface. This allows efficient implementation of other algorithms
+ * outside this class. It also allows one to efficiently implement
+ * alternate serialization or interchange formats.
+ *
+ * OpenBitSet is faster than java.util.BitSet in most operations
+ * and *much* faster at calculating cardinality of sets and results of set operations.
+ * It can also handle sets of larger cardinality (up to 64 * 2**32-1)
+ *
+ * The goals of OpenBitSet are the fastest implementation possible, and
+ * maximum code reuse. Extra safety and encapsulation
+ * may always be built on top, but if that's built in, the cost can never be removed (and
+ * hence people re-implement their own version in order to get better performance).
+ * If you want a "safe", totally encapsulated (and slower and limited) BitSet
+ * class, use java.util.BitSet.
+ *
+ * Performance Results
+ *
+ Test system: Pentium 4, Sun Java 1.5_06 -server -Xbatch -Xmx64M
+
BitSet size = 1,000,000
+
Results are java.util.BitSet time divided by OpenBitSet time.
+
+
+ | cardinality | intersect_count | union | nextSetBit | get | iterator |
+
+
+ | 50% full | 3.36 | 3.96 | 1.44 | 1.46 | 1.99 | 1.58 |
+
+
+ | 1% full | 3.31 | 3.90 | | 1.04 | | 0.99 |
+
+
+
+Test system: AMD Opteron, 64 bit linux, Sun Java 1.5_06 -server -Xbatch -Xmx64M
+
BitSet size = 1,000,000
+
Results are java.util.BitSet time divided by OpenBitSet time.
+
+
+ | cardinality | intersect_count | union | nextSetBit | get | iterator |
+
+
+ | 50% full | 2.50 | 3.50 | 1.00 | 1.03 | 1.12 | 1.25 |
+
+
+ | 1% full | 2.51 | 3.49 | | 1.00 | | 1.02 |
+
+
+
+ * @version $Id$
+ */
+
+public class OpenBitSet implements Cloneable, Serializable {
+ protected long[] bits;
+ protected int wlen; // number of words (elements) used in the array
+
+ /** Constructs an OpenBitSet large enough to hold numBits.
+ *
+ * @param numBits
+ */
+ public OpenBitSet(long numBits) {
+ bits = new long[bits2words(numBits)];
+ wlen = bits.length;
+ }
+
+ public OpenBitSet() {
+ this(64);
+ }
+
+ /** Constructs an OpenBitSet from an existing long[].
+ *
+ * The first 64 bits are in long[0],
+ * with bit index 0 at the least significant bit, and bit index 63 at the most significant.
+ * Given a bit index,
+ * the word containing it is long[index/64], and it is at bit number index%64 within that word.
+ *
+ * numWords are the number of elements in the array that contain
+ * set bits (non-zero longs).
+ * numWords should be <= bits.length, and
+ * any existing words in the array at position >= numWords should be zero.
+ *
+ */
+ public OpenBitSet(long[] bits, int numWords) {
+ this.bits = bits;
+ this.wlen = numWords;
+ }
+
+ /** Returns the current capacity in bits (1 greater than the index of the last bit) */
+ public long capacity() { return bits.length << 6; }
+
+ /**
+ * Returns the current capacity of this set. Included for
+ * compatibility. This is *not* equal to {@link #cardinality}
+ */
+ public long size() {
+ return capacity();
+ }
+
+ /** Returns true if there are no set bits */
+ public boolean isEmpty() { return cardinality()==0; }
+
+ /** Expert: returns the long[] storing the bits */
+ public long[] getBits() { return bits; }
+
+ /** Expert: sets a new long[] to use as the bit storage */
+ public void setBits(long[] bits) { this.bits = bits; }
+
+ /** Expert: gets the number of longs in the array that are in use */
+ public int getNumWords() { return wlen; }
+
+ /** Expert: sets the number of longs in the array that are in use */
+ public void setNumWords(int nWords) { this.wlen=nWords; }
+
+
+
+ /** Returns true or false for the specified bit index. */
+ public boolean get(int index) {
+ int i = index >> 6; // div 64
+ // signed shift will keep a negative index and force an
+ // array-index-out-of-bounds-exception, removing the need for an explicit check.
+ if (i>=bits.length) return false;
+
+ int bit = index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ return (bits[i] & bitmask) != 0;
+ }
+
+
+ /** Returns true or false for the specified bit index.
+ * The index should be less than the OpenBitSet size
+ */
+ public boolean fastGet(int index) {
+ int i = index >> 6; // div 64
+ // signed shift will keep a negative index and force an
+ // array-index-out-of-bounds-exception, removing the need for an explicit check.
+ int bit = index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ return (bits[i] & bitmask) != 0;
+ }
+
+
+
+ /** Returns true or false for the specified bit index
+ * The index should be less than the OpenBitSet size
+ */
+ public boolean get(long index) {
+ int i = (int)(index >> 6); // div 64
+ if (i>=bits.length) return false;
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ return (bits[i] & bitmask) != 0;
+ }
+
+ /** Returns true or false for the specified bit index. Allows specifying
+ * an index outside the current size. */
+ public boolean fastGet(long index) {
+ int i = (int)(index >> 6); // div 64
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ return (bits[i] & bitmask) != 0;
+ }
+
+ /*
+ // alternate implementation of get()
+ public boolean get1(int index) {
+ int i = index >> 6; // div 64
+ int bit = index & 0x3f; // mod 64
+ return ((bits[i]>>>bit) & 0x01) != 0;
+ // this does a long shift and a bittest (on x86) vs
+ // a long shift, and a long AND, (the test for zero is prob a no-op)
+ // testing on a P4 indicates this is slower than (bits[i] & bitmask) != 0;
+ }
+ */
+
+
+ /** returns 1 if the bit is set, 0 if not.
+ * The index should be less than the OpenBitSet size
+ */
+ public int getBit(int index) {
+ int i = index >> 6; // div 64
+ int bit = index & 0x3f; // mod 64
+ return ((int)(bits[i]>>>bit)) & 0x01;
+ }
+
+
+ /*
+ public boolean get2(int index) {
+ int word = index >> 6; // div 64
+ int bit = index & 0x0000003f; // mod 64
+ return (bits[word] << bit) < 0; // hmmm, this would work if bit order were reversed
+ // we could right shift and check for parity bit, if it was available to us.
+ }
+ */
+
+ /** sets a bit, expanding the set size if necessary */
+ public void set(long index) {
+ int wordNum = expandingWordNum(index);
+ int bit = (int)index & 0x3f;
+ long bitmask = 1L << bit;
+ bits[wordNum] |= bitmask;
+ }
+
+
+ /** Sets the bit at the specified index.
+ * The index should be less than the OpenBitSet size.
+ */
+ public void fastSet(int index) {
+ int wordNum = index >> 6; // div 64
+ int bit = index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] |= bitmask;
+ }
+
+ /** Sets the bit at the specified index.
+ * The index should be less than the OpenBitSet size.
+ */
+ public void fastSet(long index) {
+ int wordNum = (int)(index >> 6);
+ int bit = (int)index & 0x3f;
+ long bitmask = 1L << bit;
+ bits[wordNum] |= bitmask;
+ }
+
+ /** Sets a range of bits, expanding the set size if necessary
+ *
+ * @param startIndex lower index
+ * @param endIndex one-past the last bit to set
+ */
+ public void set(long startIndex, long endIndex) {
+ if (endIndex <= startIndex) return;
+
+ int startWord = (int)(startIndex>>6);
+
+ // since endIndex is one past the end, this is index of the last
+ // word to be changed.
+ int endWord = expandingWordNum(endIndex-1);
+
+ long startmask = -1L << startIndex;
+ long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
+
+ if (startWord == endWord) {
+ bits[startWord] |= (startmask & endmask);
+ return;
+ }
+
+ bits[startWord] |= startmask;
+ Arrays.fill(bits, startWord+1, endWord, -1L);
+ bits[endWord] |= endmask;
+ }
+
+
+
+ protected int expandingWordNum(long index) {
+ int wordNum = (int)(index >> 6);
+ if (wordNum>=wlen) {
+ ensureCapacity(index+1);
+ wlen = wordNum+1;
+ }
+ return wordNum;
+ }
+
+
+ /** clears a bit.
+ * The index should be less than the OpenBitSet size.
+ */
+ public void fastClear(int index) {
+ int wordNum = index >> 6;
+ int bit = index & 0x03f;
+ long bitmask = 1L << bit;
+ bits[wordNum] &= ~bitmask;
+ // hmmm, it takes one more instruction to clear than it does to set... any
+ // way to work around this? If there were only 63 bits per word, we could
+ // use a right shift of 10111111...111 in binary to position the 0 in the
+ // correct place (using sign extension).
+ // Could also use Long.rotateRight() or rotateLeft() *if* they were converted
+ // by the JVM into a native instruction.
+ // bits[word] &= Long.rotateLeft(0xfffffffe,bit);
+ }
+
+ /** clears a bit.
+ * The index should be less than the OpenBitSet size.
+ */
+ public void fastClear(long index) {
+ int wordNum = (int)(index >> 6); // div 64
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] &= ~bitmask;
+ }
+
+ /** clears a bit, allowing access beyond the current set size without changing the size.*/
+ public void clear(long index) {
+ int wordNum = (int)(index >> 6); // div 64
+ if (wordNum>=wlen) return;
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] &= ~bitmask;
+ }
+
+ /** Clears a range of bits. Clearing past the end does not change the size of the set.
+ *
+ * @param startIndex lower index
+ * @param endIndex one-past the last bit to clear
+ */
+ public void clear(long startIndex, long endIndex) {
+ if (endIndex <= startIndex) return;
+
+ int startWord = (int)(startIndex>>6);
+ if (startWord >= wlen) return;
+
+ // since endIndex is one past the end, this is index of the last
+ // word to be changed.
+ int endWord = (int)((endIndex-1)>>6);
+
+ long startmask = -1L << startIndex;
+ long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
+
+ // invert masks since we are clearing
+ startmask = ~startmask;
+ endmask = ~endmask;
+
+ if (startWord == endWord) {
+ bits[startWord] &= (startmask | endmask);
+ return;
+ }
+
+ bits[startWord] &= startmask;
+
+ int middle = Math.min(wlen, endWord);
+ Arrays.fill(bits, startWord+1, middle, 0L);
+ if (endWord < wlen) {
+ bits[endWord] &= endmask;
+ }
+ }
+
+
+
+ /** Sets a bit and returns the previous value.
+ * The index should be less than the OpenBitSet size.
+ */
+ public boolean getAndSet(int index) {
+ int wordNum = index >> 6; // div 64
+ int bit = index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ boolean val = (bits[wordNum] & bitmask) != 0;
+ bits[wordNum] |= bitmask;
+ return val;
+ }
+
+ /** Sets a bit and returns the previous value.
+ * The index should be less than the OpenBitSet size.
+ */
+ public boolean getAndSet(long index) {
+ int wordNum = (int)(index >> 6); // div 64
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ boolean val = (bits[wordNum] & bitmask) != 0;
+ bits[wordNum] |= bitmask;
+ return val;
+ }
+
+ /** flips a bit.
+ * The index should be less than the OpenBitSet size.
+ */
+ public void fastFlip(int index) {
+ int wordNum = index >> 6; // div 64
+ int bit = index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] ^= bitmask;
+ }
+
+ /** flips a bit.
+ * The index should be less than the OpenBitSet size.
+ */
+ public void fastFlip(long index) {
+ int wordNum = (int)(index >> 6); // div 64
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] ^= bitmask;
+ }
+
+ /** flips a bit, expanding the set size if necessary */
+ public void flip(long index) {
+ int wordNum = expandingWordNum(index);
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] ^= bitmask;
+ }
+
+ /** flips a bit and returns the resulting bit value.
+ * The index should be less than the OpenBitSet size.
+ */
+ public boolean flipAndGet(int index) {
+ int wordNum = index >> 6; // div 64
+ int bit = index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] ^= bitmask;
+ return (bits[wordNum] & bitmask) != 0;
+ }
+
+ /** flips a bit and returns the resulting bit value.
+ * The index should be less than the OpenBitSet size.
+ */
+ public boolean flipAndGet(long index) {
+ int wordNum = (int)(index >> 6); // div 64
+ int bit = (int)index & 0x3f; // mod 64
+ long bitmask = 1L << bit;
+ bits[wordNum] ^= bitmask;
+ return (bits[wordNum] & bitmask) != 0;
+ }
+
+ /** Flips a range of bits, expanding the set size if necessary
+ *
+ * @param startIndex lower index
+ * @param endIndex one-past the last bit to flip
+ */
+ public void flip(long startIndex, long endIndex) {
+ if (endIndex <= startIndex) return;
+ int oldlen = wlen;
+ int startWord = (int)(startIndex>>6);
+
+ // since endIndex is one past the end, this is index of the last
+ // word to be changed.
+ int endWord = expandingWordNum(endIndex-1);
+
+ /*** Java shifting wraps around so -1L>>>64 == -1 .
+ * For that reason, make sure not to use endmask if the bits to flip will
+ * be zero in the last word (redefine endWord to be the last changed...)
+ long startmask = -1L << (startIndex & 0x3f); // example: 11111...111000
+ long endmask = -1L >>> (64-(endIndex & 0x3f)); // example: 00111...111111
+ ***/
+
+ long startmask = -1L << startIndex;
+ long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
+
+ if (startWord == endWord) {
+ bits[startWord] ^= (startmask & endmask);
+ return;
+ }
+
+ bits[startWord] ^= startmask;
+
+ for (int i=startWord+1; i b.wlen) {
+ tot += BitUtil.pop_array(a.bits, b.wlen, a.wlen-b.wlen);
+ }
+ return tot;
+ }
+
+ /** Returns the popcount or cardinality of "a and not b"
+ * or "intersection(a, not(b))".
+ * Neither set is modified.
+ */
+ public static long andNotCount(OpenBitSet a, OpenBitSet b) {
+ long tot = BitUtil.pop_andnot(a.bits, b.bits, 0, Math.min(a.wlen, b.wlen));
+ if (a.wlen > b.wlen) {
+ tot += BitUtil.pop_array(a.bits, b.wlen, a.wlen-b.wlen);
+ }
+ return tot;
+ }
+
+ /** Returns the popcount or cardinality of the exclusive-or of the two sets.
+ * Neither set is modified.
+ */
+ public static long xorCount(OpenBitSet a, OpenBitSet b) {
+ long tot = BitUtil.pop_xor(a.bits, b.bits, 0, Math.min(a.wlen, b.wlen));
+ if (a.wlen < b.wlen) {
+ tot += BitUtil.pop_array(b.bits, a.wlen, b.wlen-a.wlen);
+ } else if (a.wlen > b.wlen) {
+ tot += BitUtil.pop_array(a.bits, b.wlen, a.wlen-b.wlen);
+ }
+ return tot;
+ }
+
+
+ /** Returns the index of the first set bit starting at the index specified.
+ * -1 is returned if there are no more set bits.
+ */
+ public int nextSetBit(int index) {
+ int i = index>>6;
+ if (i>=wlen) return -1;
+ int subIndex = index & 0x3f; // index within the word
+ long word = bits[i] >> subIndex; // skip all the bits to the right of index
+
+ if (word!=0) {
+ return (i<<6) + subIndex + BitUtil.ntz(word);
+ }
+
+ while(++i < wlen) {
+ word = bits[i];
+ if (word!=0) return (i<<6) + BitUtil.ntz(word);
+ }
+
+ return -1;
+ }
+
+ /** Returns the index of the first set bit starting at the index specified.
+ * -1 is returned if there are no more set bits.
+ */
+ public long nextSetBit(long index) {
+ int i = (int)(index>>>6);
+ if (i>=wlen) return -1;
+ int subIndex = (int)index & 0x3f; // index within the word
+ long word = bits[i] >>> subIndex; // skip all the bits to the right of index
+
+ if (word!=0) {
+ return (((long)i)<<6) + (subIndex + BitUtil.ntz(word));
+ }
+
+ while(++i < wlen) {
+ word = bits[i];
+ if (word!=0) return (((long)i)<<6) + BitUtil.ntz(word);
+ }
+
+ return -1;
+ }
+
+
+
+
+ public Object clone() {
+ try {
+ OpenBitSet obs = (OpenBitSet)super.clone();
+ obs.bits = (long[]) obs.bits.clone(); // hopefully an array clone is as fast(er) than arraycopy
+ return obs;
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** this = this AND other */
+ public void intersect(OpenBitSet other) {
+ int newLen= Math.min(this.wlen,other.wlen);
+ long[] thisArr = this.bits;
+ long[] otherArr = other.bits;
+ // testing against zero can be more efficient
+ int pos=newLen;
+ while(--pos>=0) {
+ thisArr[pos] &= otherArr[pos];
+ }
+ if (this.wlen > newLen) {
+ // fill zeros from the new shorter length to the old length
+ Arrays.fill(bits,newLen,this.wlen,0);
+ }
+ this.wlen = newLen;
+ }
+
+ /** this = this OR other */
+ public void union(OpenBitSet other) {
+ int newLen = Math.max(wlen,other.wlen);
+ ensureCapacityWords(newLen);
+
+ long[] thisArr = this.bits;
+ long[] otherArr = other.bits;
+ int pos=Math.min(wlen,other.wlen);
+ while(--pos>=0) {
+ thisArr[pos] |= otherArr[pos];
+ }
+ if (this.wlen < newLen) {
+ System.arraycopy(otherArr, this.wlen, thisArr, this.wlen, newLen-this.wlen);
+ }
+ this.wlen = newLen;
+ }
+
+
+ /** Remove all elements set in other. this = this AND_NOT other */
+ public void remove(OpenBitSet other) {
+ int idx = Math.min(wlen,other.wlen);
+ long[] thisArr = this.bits;
+ long[] otherArr = other.bits;
+ while(--idx>=0) {
+ thisArr[idx] &= ~otherArr[idx];
+ }
+ }
+
+ /** this = this XOR other */
+ public void xor(OpenBitSet other) {
+ int newLen = Math.max(wlen,other.wlen);
+ ensureCapacityWords(newLen);
+
+ long[] thisArr = this.bits;
+ long[] otherArr = other.bits;
+ int pos=Math.min(wlen,other.wlen);
+ while(--pos>=0) {
+ thisArr[pos] ^= otherArr[pos];
+ }
+ if (this.wlen < newLen) {
+ System.arraycopy(otherArr, this.wlen, thisArr, this.wlen, newLen-this.wlen);
+ }
+ this.wlen = newLen;
+ }
+
+
+ // some BitSet compatability methods
+
+ //** see {@link intersect} */
+ public void and(OpenBitSet other) {
+ intersect(other);
+ }
+
+ //** see {@link union} */
+ public void or(OpenBitSet other) {
+ union(other);
+ }
+
+ //** see {@link andNot} */
+ public void andNot(OpenBitSet other) {
+ remove(other);
+ }
+
+ /** returns true if the sets have any elements in common */
+ public boolean intersects(OpenBitSet other) {
+ int pos = Math.min(this.wlen, other.wlen);
+ long[] thisArr = this.bits;
+ long[] otherArr = other.bits;
+ while (--pos>=0) {
+ if ((thisArr[pos] & otherArr[pos])!=0) return true;
+ }
+ return false;
+ }
+
+
+
+ /** Expand the long[] with the size given as a number of words (64 bit longs).
+ * getNumWords() is unchanged by this call.
+ */
+ public void ensureCapacityWords(int numWords) {
+ if (bits.length < numWords) {
+ long[] newBits = new long[numWords];
+ System.arraycopy(bits,0,newBits,0,wlen);
+ bits = newBits;
+ }
+ }
+
+ /** Ensure that the long[] is big enough to hold numBits, expanding it if necessary.
+ * getNumWords() is unchanged by this call.
+ */
+ public void ensureCapacity(long numBits) {
+ ensureCapacityWords(bits2words(numBits));
+ }
+
+ /** Lowers numWords, the number of words in use,
+ * by checking for trailing zero words.
+ */
+ public void trimTrailingZeros() {
+ int idx = wlen-1;
+ while (idx>=0 && bits[idx]==0) idx--;
+ wlen = idx+1;
+ }
+
+ /** returns the number of 64 bit words it would take to hold numBits */
+ public static int bits2words(long numBits) {
+ return (int)(((numBits-1)>>>6)+1);
+ }
+
+
+ /** returns true if both sets have the same bits set */
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof OpenBitSet)) return false;
+ OpenBitSet a;
+ OpenBitSet b = (OpenBitSet)o;
+ // make a the larger set.
+ if (b.wlen > this.wlen) {
+ a = b; b=this;
+ } else {
+ a=this;
+ }
+
+ // check for any set bits out of the range of b
+ for (int i=a.wlen-1; i>=b.wlen; i--) {
+ if (a.bits[i]!=0) return false;
+ }
+
+ for (int i=b.wlen-1; i>=0; i--) {
+ if (a.bits[i] != b.bits[i]) return false;
+ }
+
+ return true;
+ }
+
+
+ public int hashCode() {
+ long h = 0x98761234; // something non-zero for length==0
+ for (int i = bits.length; --i>=0;) {
+ h ^= bits[i];
+ h = (h << 1) | (h >>> 31); // rotate left
+ }
+ return (int)((h>>32) ^ h); // fold leftmost bits into right
+ }
+
+}
+
+
Property changes on: src/java/org/apache/lucene/util/OpenBitSet.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/java/org/apache/lucene/util/OpenBitSetMatcher.java
===================================================================
--- src/java/org/apache/lucene/util/OpenBitSetMatcher.java (revision 0)
+++ src/java/org/apache/lucene/util/OpenBitSetMatcher.java (revision 0)
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util; // started from org.apache.solr.util.BitSetIterator, rev. 555343
+
+import org.apache.lucene.search.Matcher;
+
+/** An iterator to iterate over set bits in an OpenBitSet.
+ * This is faster than nextSetBit() for iterating over the complete set of bits,
+ * especially when the density of the bits set is high.
+ *
+ * @version $Id$
+ */
+public class OpenBitSetMatcher extends Matcher {
+
+ // The General Idea: instead of having an array per byte that has
+ // the offsets of the next set bit, that array could be
+ // packed inside a 32 bit integer (8 4 bit numbers). That
+ // should be faster than accessing an array for each index, and
+ // the total array size is kept smaller (256*sizeof(int))=1K
+ protected final static int[] bitlist={
+ 0x0,0x1,0x2,0x21,0x3,0x31,0x32,0x321,0x4,0x41,0x42,0x421,0x43,0x431,0x432,0x4321,0x5,0x51,0x52,0x521,0x53,0x531,0x532,0x5321,0x54,0x541,0x542,0x5421,0x543,0x5431,0x5432,0x54321,0x6,0x61,0x62,0x621,0x63,0x631,0x632,0x6321,0x64,0x641,0x642,0x6421,0x643,0x6431,0x6432,0x64321,0x65,0x651,0x652,0x6521,0x653,0x6531,0x6532,0x65321,0x654,0x6541,0x6542,0x65421,0x6543,0x65431,0x65432,0x654321,0x7,0x71,0x72,0x721,0x73,0x731,0x732,0x7321,0x74,0x741,0x742,0x7421,0x743,0x7431,0x7432,0x74321,0x75,0x751,0x752,0x7521,0x753,0x7531,0x7532,0x75321,0x754,0x7541,0x7542,0x75421,0x7543,0x75431,0x75432,0x754321,0x76,0x761,0x762,0x7621,0x763,0x7631,0x7632,0x76321,0x764,0x7641,0x7642,0x76421,0x7643,0x76431,0x76432,0x764321,0x765,0x7651,0x7652,0x76521,0x7653,0x76531,0x76532,0x765321,0x7654,0x76541,0x76542,0x765421,0x76543,0x765431,0x765432,0x7654321,0x8,0x81,0x82,0x821,0x83,0x831,0x832,0x8321,0x84,0x841,0x842,0x8421,0x843,0x8431,0x8432,0x84321,0x85,0x851,0x852,0x8521,0x853,0x8531,0x8532,0x85321,0x854,0x8541,0x8542,0x85421,0x8543,0x85431,0x85432,0x854321,0x86,0x861,0x862,0x8621,0x863,0x8631,0x8632,0x86321,0x864,0x8641,0x8642,0x86421,0x8643,0x86431,0x86432,0x864321,0x865,0x8651,0x8652,0x86521,0x8653,0x86531,0x86532,0x865321,0x8654,0x86541,0x86542,0x865421,0x86543,0x865431,0x865432,0x8654321,0x87,0x871,0x872,0x8721,0x873,0x8731,0x8732,0x87321,0x874,0x8741,0x8742,0x87421,0x8743,0x87431,0x87432,0x874321,0x875,0x8751,0x8752,0x87521,0x8753,0x87531,0x87532,0x875321,0x8754,0x87541,0x87542,0x875421,0x87543,0x875431,0x875432,0x8754321,0x876,0x8761,0x8762,0x87621,0x8763,0x87631,0x87632,0x876321,0x8764,0x87641,0x87642,0x876421,0x87643,0x876431,0x876432,0x8764321,0x8765,0x87651,0x87652,0x876521,0x87653,0x876531,0x876532,0x8765321,0x87654,0x876541,0x876542,0x8765421,0x876543,0x8765431,0x8765432,0x87654321
+ };
+ /***** the python code that generated bitlist
+ def bits2int(val):
+ arr=0
+ for shift in range(8,0,-1):
+ if val & 0x80:
+ arr = (arr << 4) | shift
+ val = val << 1
+ return arr
+
+ def int_table():
+ tbl = [ hex(bits2int(val)).strip('L') for val in range(256) ]
+ return ','.join(tbl)
+ ******/
+
+ // hmmm, what about an iterator that finds zeros though,
+ // or a reverse iterator... should they be separate classes
+ // for efficiency, or have a common root interface? (or
+ // maybe both? could ask for a SetBitsIterator, etc...
+
+
+ private final long[] arr;
+ private final int words;
+ private int i=-1;
+ private long word;
+ private int wordShift;
+ private int indexArray;
+
+ public OpenBitSetMatcher(OpenBitSet obs) {
+ this(obs.getBits(), obs.getNumWords());
+ }
+
+ public OpenBitSetMatcher(long[] bits, int numWords) {
+ arr = bits;
+ words = numWords;
+ }
+
+ // 64 bit shifts
+ private void shift() {
+ if ((int)word ==0) {wordShift +=32; word = word >>>32; }
+ if ((word & 0x0000FFFF) == 0) { wordShift +=16; word >>>=16; }
+ if ((word & 0x000000FF) == 0) { wordShift +=8; word >>>=8; }
+ indexArray = bitlist[(int)word & 0xff];
+ }
+
+ /***** alternate shift implementations
+ // 32 bit shifts, but a long shift needed at the end
+ private void shift2() {
+ int y = (int)word;
+ if (y==0) {wordShift +=32; y = (int)(word >>>32); }
+ if ((y & 0x0000FFFF) == 0) { wordShift +=16; y>>>=16; }
+ if ((y & 0x000000FF) == 0) { wordShift +=8; y>>>=8; }
+ indexArray = bitlist[y & 0xff];
+ word >>>= (wordShift +1);
+ }
+
+ private void shift3() {
+ int lower = (int)word;
+ int lowByte = lower & 0xff;
+ if (lowByte != 0) {
+ indexArray=bitlist[lowByte];
+ return;
+ }
+ shift();
+ }
+ ******/
+
+ private int _next() {
+ if (indexArray==0) {
+ if (word!=0) {
+ word >>>= 8;
+ wordShift += 8;
+ }
+
+ while (word==0) {
+ if (++i >= words) return -1;
+ word = arr[i];
+ wordShift =-1; // loop invariant code motion should move this
+ }
+
+ // after the first time, should I go with a linear search, or
+ // stick with the binary search in shift?
+ shift();
+ }
+
+ int bitIndex = (indexArray & 0x0f) + wordShift;
+ indexArray >>>= 4;
+ // should i<<6 be cached as a separate variable?
+ // it would only save one cycle in the best circumstances.
+ return (i<<6) + bitIndex;
+ }
+
+ private int _next(int fromIndex) {
+ indexArray=0;
+ i = fromIndex >> 6;
+ if (i>=words) {
+ word =0; // setup so _next() will also return -1
+ return -1;
+ }
+ wordShift = fromIndex & 0x3f;
+ word = arr[i] >>> wordShift;
+ if (word !=0) {
+ wordShift--; // compensate for 1 based arrIndex
+ } else {
+ while (word ==0) {
+ if (++i >= words) return -1;
+ word = arr[i];
+ }
+ wordShift =-1;
+ }
+
+ shift();
+
+ int bitIndex = (indexArray & 0x0f) + wordShift;
+ indexArray >>>= 4;
+ // should i<<6 be cached as a separate variable?
+ // it would only save one cycle in the best circumstances.
+ return (i<<6) + bitIndex;
+ }
+
+ /* To be implemented for Matcher: next(), doc(), skipTo(), renamed solr next() to _next() */
+
+ /** Advances to the next matching document.
+ *
When this method is used the {@link #explain(int)}
+ * and {@link #match(MatchCollector)} methods should not be used.
+ * @return true iff there is another matching document.
+ */
+ public boolean next() {
+ return _next() != -1;
+ }
+
+ /** Returns the current matching document number.
+ * Initially invalid, until {@link #next()} or {@link #skipTo(int)}
+ * is called the first time.
+ */
+ public int doc() {
+ int bitIndex = (indexArray & 0x0f) + wordShift;
+ return (i<<6) + bitIndex;
+ }
+
+ /** Skips to the first match whose document number is greater than
+ * or equal to a given target.
+ * If, after {@link #next()} or {@link #skipTo(int)}
+ * has been called the first time,
+ * the target is before or at the current document,
+ * the current document may change to the next matching document.
+ *
When this method is used the {@link #explain(int)}
+ * and {@link #match(MatchCollector)} methods should not be used.
+ * @param target The target document number.
+ * @return true iff there is such a match.
+ */
+ public boolean skipTo(int target) {
+ return _next(target) != -1;
+ }
+}
Property changes on: src/java/org/apache/lucene/util/OpenBitSetMatcher.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/java/org/apache/lucene/util/BitUtil.java
===================================================================
--- src/java/org/apache/lucene/util/BitUtil.java (revision 0)
+++ src/java/org/apache/lucene/util/BitUtil.java (revision 0)
@@ -0,0 +1,799 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util; // from org.apache.solr.util rev 555343
+
+/** A variety of high efficiencly bit twiddling routines.
+ *
+ * @version $Id$
+ */
+public class BitUtil {
+
+ /** Returns the number of bits set in the long */
+ public static int pop(long x) {
+ /* Hacker's Delight 32 bit pop function:
+ * http://www.hackersdelight.org/HDcode/newCode/pop_arrayHS.cc
+ *
+ int pop(unsigned x) {
+ x = x - ((x >> 1) & 0x55555555);
+ x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
+ x = (x + (x >> 4)) & 0x0F0F0F0F;
+ x = x + (x >> 8);
+ x = x + (x >> 16);
+ return x & 0x0000003F;
+ }
+ ***/
+
+ // 64 bit java version of the C function from above
+ x = x - ((x >>> 1) & 0x5555555555555555L);
+ x = (x & 0x3333333333333333L) + ((x >>>2 ) & 0x3333333333333333L);
+ x = (x + (x >>> 4)) & 0x0F0F0F0F0F0F0F0FL;
+ x = x + (x >>> 8);
+ x = x + (x >>> 16);
+ x = x + (x >>> 32);
+ return ((int)x) & 0x7F;
+ }
+
+ /*** Returns the number of set bits in an array of longs. */
+ public static long pop_array(long A[], int wordOffset, int numWords) {
+ /*
+ * Robert Harley and David Seal's bit counting algorithm, as documented
+ * in the revisions of Hacker's Delight
+ * http://www.hackersdelight.org/revisions.pdf
+ * http://www.hackersdelight.org/HDcode/newCode/pop_arrayHS.cc
+ *
+ * This function was adapted to Java, and extended to use 64 bit words.
+ * if only we had access to wider registers like SSE from java...
+ *
+ * This function can be transformed to compute the popcount of other functions
+ * on bitsets via something like this:
+ * sed 's/A\[\([^]]*\)\]/\(A[\1] \& B[\1]\)/g'
+ *
+ */
+ int n = wordOffset+numWords;
+ long tot=0, tot8=0;
+ long ones=0, twos=0, fours=0;
+
+ int i;
+ for (i = wordOffset; i <= n - 8; i+=8) {
+ /*** C macro from Hacker's Delight
+ #define CSA(h,l, a,b,c) \
+ {unsigned u = a ^ b; unsigned v = c; \
+ h = (a & b) | (u & v); l = u ^ v;}
+ ***/
+
+ long twosA,twosB,foursA,foursB,eights;
+
+ // CSA(twosA, ones, ones, A[i], A[i+1])
+ {
+ long b=A[i], c=A[i+1];
+ long u=ones ^ b;
+ twosA=(ones & b)|( u & c);
+ ones=u^c;
+ }
+ // CSA(twosB, ones, ones, A[i+2], A[i+3])
+ {
+ long b=A[i+2], c=A[i+3];
+ long u=ones^b;
+ twosB =(ones&b)|(u&c);
+ ones=u^c;
+ }
+ //CSA(foursA, twos, twos, twosA, twosB)
+ {
+ long u=twos^twosA;
+ foursA=(twos&twosA)|(u&twosB);
+ twos=u^twosB;
+ }
+ //CSA(twosA, ones, ones, A[i+4], A[i+5])
+ {
+ long b=A[i+4], c=A[i+5];
+ long u=ones^b;
+ twosA=(ones&b)|(u&c);
+ ones=u^c;
+ }
+ // CSA(twosB, ones, ones, A[i+6], A[i+7])
+ {
+ long b=A[i+6], c=A[i+7];
+ long u=ones^b;
+ twosB=(ones&b)|(u&c);
+ ones=u^c;
+ }
+ //CSA(foursB, twos, twos, twosA, twosB)
+ {
+ long u=twos^twosA;
+ foursB=(twos&twosA)|(u&twosB);
+ twos=u^twosB;
+ }
+
+ //CSA(eights, fours, fours, foursA, foursB)
+ {
+ long u=fours^foursA;
+ eights=(fours&foursA)|(u&foursB);
+ fours=u^foursB;
+ }
+ tot8 += pop(eights);
+ }
+
+ // handle trailing words in a binary-search manner...
+ // derived from the loop above by setting specific elements to 0.
+ // the original method in Hackers Delight used a simple for loop:
+ // for (i = i; i < n; i++) // Add in the last elements
+ // tot = tot + pop(A[i]);
+
+ if (i<=n-4) {
+ long twosA, twosB, foursA, eights;
+ {
+ long b=A[i], c=A[i+1];
+ long u=ones ^ b;
+ twosA=(ones & b)|( u & c);
+ ones=u^c;
+ }
+ {
+ long b=A[i+2], c=A[i+3];
+ long u=ones^b;
+ twosB =(ones&b)|(u&c);
+ ones=u^c;
+ }
+ {
+ long u=twos^twosA;
+ foursA=(twos&twosA)|(u&twosB);
+ twos=u^twosB;
+ }
+ eights=fours&foursA;
+ fours=fours^foursA;
+
+ tot8 += pop(eights);
+ i+=4;
+ }
+
+ if (i<=n-2) {
+ long b=A[i], c=A[i+1];
+ long u=ones ^ b;
+ long twosA=(ones & b)|( u & c);
+ ones=u^c;
+
+ long foursA=twos&twosA;
+ twos=twos^twosA;
+
+ long eights=fours&foursA;
+ fours=fours^foursA;
+
+ tot8 += pop(eights);
+ i+=2;
+ }
+
+ if (i>= 1
+ return i
+ print ','.join([ str(ntz(i)) for i in range(256) ])
+ ***/
+ /** table of number of trailing zeros in a byte */
+ public static final byte[] ntzTable = {8,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,5,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,6,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,5,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,7,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,5,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,6,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,5,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0,4,0,1,0,2,0,1,0,3,0,1,0,2,0,1,0};
+
+
+ /** Returns number of trailing zeros in the 64 bit long value. */
+ public static int ntz(long val) {
+ // A full binary search to determine the low byte was slower than
+ // a linear search for nextSetBit(). This is most likely because
+ // the implementation of nextSetBit() shifts bits to the right, increasing
+ // the probability that the first non-zero byte is in the rhs.
+ //
+ // This implementation does a single binary search at the top level only
+ // so that all other bit shifting can be done on ints instead of longs to
+ // remain friendly to 32 bit architectures. In addition, the case of a
+ // non-zero first byte is checked for first because it is the most common
+ // in dense bit arrays.
+
+ int lower = (int)val;
+ int lowByte = lower & 0xff;
+ if (lowByte != 0) return ntzTable[lowByte];
+
+ if (lower!=0) {
+ lowByte = (lower>>>8) & 0xff;
+ if (lowByte != 0) return ntzTable[lowByte] + 8;
+ lowByte = (lower>>>16) & 0xff;
+ if (lowByte != 0) return ntzTable[lowByte] + 16;
+ // no need to mask off low byte for the last byte in the 32 bit word
+ // no need to check for zero on the last byte either.
+ return ntzTable[lower>>>24] + 24;
+ } else {
+ // grab upper 32 bits
+ int upper=(int)(val>>32);
+ lowByte = upper & 0xff;
+ if (lowByte != 0) return ntzTable[lowByte] + 32;
+ lowByte = (upper>>>8) & 0xff;
+ if (lowByte != 0) return ntzTable[lowByte] + 40;
+ lowByte = (upper>>>16) & 0xff;
+ if (lowByte != 0) return ntzTable[lowByte] + 48;
+ // no need to mask off low byte for the last byte in the 32 bit word
+ // no need to check for zero on the last byte either.
+ return ntzTable[upper>>>24] + 56;
+ }
+ }
+
+ /** returns 0 based index of first set bit
+ * (only works for x!=0)
+ *
This is an alternate implementation of ntz()
+ */
+ public static int ntz2(long x) {
+ int n = 0;
+ int y = (int)x;
+ if (y==0) {n+=32; y = (int)(x>>>32); } // the only 64 bit shift necessary
+ if ((y & 0x0000FFFF) == 0) { n+=16; y>>>=16; }
+ if ((y & 0x000000FF) == 0) { n+=8; y>>>=8; }
+ return (ntzTable[ y & 0xff ]) + n;
+ }
+
+ /** returns 0 based index of first set bit
+ *
This is an alternate implementation of ntz()
+ */
+ public static int ntz3(long x) {
+ // another implementation taken from Hackers Delight, extended to 64 bits
+ // and converted to Java.
+ // Many 32 bit ntz algorithms are at http://www.hackersdelight.org/HDcode/ntz.cc
+ int n = 1;
+
+ // do the first step as a long, all others as ints.
+ int y = (int)x;
+ if (y==0) {n+=32; y = (int)(x>>>32); }
+ if ((y & 0x0000FFFF) == 0) { n+=16; y>>>=16; }
+ if ((y & 0x000000FF) == 0) { n+=8; y>>>=8; }
+ if ((y & 0x0000000F) == 0) { n+=4; y>>>=4; }
+ if ((y & 0x00000003) == 0) { n+=2; y>>>=2; }
+ return n - (y & 1);
+ }
+
+
+ /** returns true if v is a power of two or zero*/
+ public static boolean isPowerOfTwo(int v) {
+ return ((v & (v-1)) == 0);
+ }
+
+ /** returns true if v is a power of two or zero*/
+ public static boolean isPowerOfTwo(long v) {
+ return ((v & (v-1)) == 0);
+ }
+
+ /** returns the next highest power of two, or the current value if it's already a power of two or zero*/
+ public static int nextHighestPowerOfTwo(int v) {
+ v--;
+ v |= v >> 1;
+ v |= v >> 2;
+ v |= v >> 4;
+ v |= v >> 8;
+ v |= v >> 16;
+ v++;
+ return v;
+ }
+
+ /** returns the next highest power of two, or the current value if it's already a power of two or zero*/
+ public static long nextHighestPowerOfTwo(long v) {
+ v--;
+ v |= v >> 1;
+ v |= v >> 2;
+ v |= v >> 4;
+ v |= v >> 8;
+ v |= v >> 16;
+ v |= v >> 32;
+ v++;
+ return v;
+ }
+
+}
Property changes on: src/java/org/apache/lucene/util/BitUtil.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/java/org/apache/lucene/util/SortedVIntList.java
===================================================================
--- src/java/org/apache/lucene/util/SortedVIntList.java (revision 0)
+++ src/java/org/apache/lucene/util/SortedVIntList.java (revision 0)
@@ -0,0 +1,216 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.BitSet;
+import org.apache.lucene.search.Matcher;
+
+/**
+ * Store and iterate sorted integers in compressed form in RAM.
+ *
The code for compressing the differences between ascending integers was
+ * borrowed from {@link org.apache.lucene.store.IndexInput} and
+ * {@link org.apache.lucene.store.IndexOutput}.
+ */
+public class SortedVIntList implements MatcherProvider {
+ /** When a BitSet has fewer than 1 in BITS2VINTLIST_SIZE bits set,
+ * a SortedVIntList representing the index numbers of the set bits
+ * will be smaller than that BitSet.
+ */
+ final static int BITS2VINTLIST_SIZE = 8;
+
+ private int size;
+ private byte[] bytes;
+ private int lastBytePos;
+
+ /**
+ * Create a SortedVIntList from all elements of an array of integers.
+ *
+ * @param sortedInts A sorted array of non negative integers.
+ */
+ public SortedVIntList(int[] sortedInts) {
+ this(sortedInts, sortedInts.length);
+ }
+
+ /**
+ * Create a SortedVIntList from an array of integers.
+ * @param sortedInts An array of sorted non negative integers.
+ * @param inputSize The number of integers to be used from the array.
+ */
+ public SortedVIntList(int[] sortedInts, int inputSize) {
+ SortedVIntListBuilder builder = new SortedVIntListBuilder();
+ for (int i = 0; i < inputSize; i++) {
+ builder.addInt(sortedInts[i]);
+ }
+ builder.done();
+ }
+
+ /**
+ * Create a SortedVIntList from a BitSet.
+ * @param bits A bit set representing a set of integers.
+ */
+ public SortedVIntList(BitSet bits) {
+ SortedVIntListBuilder builder = new SortedVIntListBuilder();
+ int nextInt = bits.nextSetBit(0);
+ while (nextInt != -1) {
+ builder.addInt(nextInt);
+ nextInt = bits.nextSetBit(nextInt + 1);
+ }
+ builder.done();
+ }
+
+ /**
+ * Create a SortedVIntList from an OpenBitSet.
+ * @param bits A bit set representing a set of integers.
+ */
+ public SortedVIntList(OpenBitSet bits) {
+ SortedVIntListBuilder builder = new SortedVIntListBuilder();
+ int nextInt = bits.nextSetBit(0);
+ while (nextInt != -1) {
+ builder.addInt(nextInt);
+ nextInt = bits.nextSetBit(nextInt + 1);
+ }
+ builder.done();
+ }
+
+ /**
+ * Create a SortedVIntList.
+ * @param matcher A Matcher providing document numbers as a set of integers.
+ * This Matcher is iterated completely when this constructor
+ * is called and it must provide the integers in non
+ * decreasing order.
+ */
+ public SortedVIntList(Matcher matcher) throws IOException {
+ SortedVIntListBuilder builder = new SortedVIntListBuilder();
+ while (matcher.next()) {
+ builder.addInt(matcher.doc());
+ }
+ builder.done();
+ }
+
+
+ private class SortedVIntListBuilder {
+ private int lastInt = 0;
+
+ SortedVIntListBuilder() {
+ initBytes();
+ lastInt = 0;
+ }
+
+ void addInt(int nextInt) {
+ int diff = nextInt - lastInt;
+ if (diff < 0) {
+ throw new IllegalArgumentException(
+ "Input not sorted or first element negative.");
+ }
+
+ if ((lastBytePos + MAX_BYTES_PER_INT) > bytes.length) {
+ // biggest possible int does not fit
+ resizeBytes((bytes.length * 2) + MAX_BYTES_PER_INT);
+ }
+
+ // See org.apache.lucene.store.IndexOutput.writeVInt()
+ while ((diff & ~VB1) != 0) { // The high bit of the next byte needs to be set.
+ bytes[lastBytePos++] = (byte) ((diff & VB1) | ~VB1);
+ diff >>>= BIT_SHIFT;
+ }
+ bytes[lastBytePos++] = (byte) diff; // Last byte, high bit not set.
+ size++;
+ lastInt = nextInt;
+ }
+
+ void done() {
+ resizeBytes(lastBytePos);
+ }
+ }
+
+
+ private void initBytes() {
+ size = 0;
+ bytes = new byte[128]; // initial byte size
+ lastBytePos = 0;
+ }
+
+ private void resizeBytes(int newSize) {
+ if (newSize != bytes.length) {
+ byte[] newBytes = new byte[newSize];
+ System.arraycopy(bytes, 0, newBytes, 0, lastBytePos);
+ bytes = newBytes;
+ }
+ }
+
+ private static final int VB1 = 0x7F;
+ private static final int BIT_SHIFT = 7;
+ private final int MAX_BYTES_PER_INT = (31 / BIT_SHIFT) + 1;
+
+ /**
+ * @return The total number of sorted integers.
+ */
+ public int size() {
+ return size;
+ }
+
+ /**
+ * @return The size of the byte array storing the compressed sorted integers.
+ */
+ public int getByteSize() {
+ return bytes.length;
+ }
+
+ /**
+ * @return A Matcher over the sorted integers.
+ */
+ public Matcher getMatcher() {
+ return new Matcher() {
+ int bytePos = 0;
+ int lastInt = 0;
+
+ private void advance() {
+ // See org.apache.lucene.store.IndexInput.readVInt()
+ byte b = bytes[bytePos++];
+ lastInt += b & VB1;
+ for (int s = BIT_SHIFT; (b & ~VB1) != 0; s += BIT_SHIFT) {
+ b = bytes[bytePos++];
+ lastInt += (b & VB1) << s;
+ }
+ }
+
+ public int doc() {return lastInt;}
+
+ public boolean next() {
+ if (bytePos >= lastBytePos) {
+ return false;
+ } else {
+ advance();
+ return true;
+ }
+ }
+
+ public boolean skipTo(int docNr) {
+ while (bytePos < lastBytePos) {
+ advance();
+ if (lastInt >= docNr) { // No skipping to docNr available.
+ return true;
+ }
+ }
+ return false;
+ }
+ };
+ }
+}
+
Property changes on: src/java/org/apache/lucene/util/SortedVIntList.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/test/org/apache/lucene/util/TestSortedVIntList.java
===================================================================
--- src/test/org/apache/lucene/util/TestSortedVIntList.java (revision 0)
+++ src/test/org/apache/lucene/util/TestSortedVIntList.java (revision 0)
@@ -0,0 +1,198 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+import junit.textui.TestRunner;
+
+import org.apache.lucene.search.Matcher;
+
+public class TestSortedVIntList extends TestCase {
+ /** Main for running test case by itself. */
+ public static void main(String args[]) {
+ TestRunner.run(new TestSuite(TestSortedVIntList.class));
+ }
+
+ void tstMatcher(
+ SortedVIntList vintList,
+ int[] ints) throws IOException {
+ for (int i = 0; i < ints.length; i++) {
+ if ((i > 0) && (ints[i-1] == ints[i])) {
+ return; // DocNrSkipper should not skip to same document.
+ }
+ }
+ Matcher m = vintList.getMatcher();
+ for (int i = 0; i < ints.length; i++) {
+ assertTrue("No end of Matcher at: " + i, m.next());
+ assertEquals(ints[i], m.doc());
+ }
+ assertTrue("End of Matcher", (! m.next()));
+ }
+
+ void tstVIntList(
+ SortedVIntList vintList,
+ int[] ints,
+ int expectedByteSize) throws IOException {
+ assertEquals("Size", ints.length, vintList.size());
+ assertEquals("Byte size", expectedByteSize, vintList.getByteSize());
+ tstMatcher(vintList, ints);
+ }
+
+ public void tstViaBitSet(int [] ints, int expectedByteSize) throws IOException {
+ final int MAX_INT_FOR_BITSET = 1024 * 1024;
+ BitSet bs = new BitSet();
+ for (int i = 0; i < ints.length; i++) {
+ if (ints[i] > MAX_INT_FOR_BITSET) {
+ return; // BitSet takes too much memory
+ }
+ if ((i > 0) && (ints[i-1] == ints[i])) {
+ return; // BitSet cannot store duplicate.
+ }
+ bs.set(ints[i]);
+ }
+ SortedVIntList svil = new SortedVIntList(bs);
+ tstVIntList(svil, ints, expectedByteSize);
+ tstVIntList(new SortedVIntList(svil.getMatcher()), ints, expectedByteSize);
+ }
+
+ private static final int VB1 = 0x7F;
+ private static final int BIT_SHIFT = 7;
+ private static final int VB2 = (VB1 << BIT_SHIFT) | VB1;
+ private static final int VB3 = (VB2 << BIT_SHIFT) | VB1;
+ private static final int VB4 = (VB3 << BIT_SHIFT) | VB1;
+
+ private int vIntByteSize(int i) {
+ assert i >= 0;
+ if (i <= VB1) return 1;
+ if (i <= VB2) return 2;
+ if (i <= VB3) return 3;
+ if (i <= VB4) return 4;
+ return 5;
+ }
+
+ private int vIntListByteSize(int [] ints) {
+ int byteSize = 0;
+ int last = 0;
+ for (int i = 0; i < ints.length; i++) {
+ byteSize += vIntByteSize(ints[i] - last);
+ last = ints[i];
+ }
+ return byteSize;
+ }
+
+ public void tstInts(int [] ints) {
+ int expectedByteSize = vIntListByteSize(ints);
+ try {
+ tstVIntList(new SortedVIntList(ints), ints, expectedByteSize);
+ tstViaBitSet(ints, expectedByteSize);
+ } catch (IOException ioe) {
+ throw new Error(ioe);
+ }
+ }
+
+ public void tstIllegalArgExc(int [] ints) {
+ try {
+ new SortedVIntList(ints);
+ }
+ catch (IllegalArgumentException e) {
+ return;
+ }
+ fail("Expected IllegalArgumentException");
+ }
+
+ private int[] fibArray(int a, int b, int size) {
+ final int[] fib = new int[size];
+ fib[0] = a;
+ fib[1] = b;
+ for (int i = 2; i < size; i++) {
+ fib[i] = fib[i-1] + fib[i-2];
+ }
+ return fib;
+ }
+
+ private int[] reverseDiffs(int []ints) { // reverse the order of the successive differences
+ final int[] res = new int[ints.length];
+ for (int i = 0; i < ints.length; i++) {
+ res[i] = ints[ints.length - 1] + (ints[0] - ints[ints.length - 1 - i]);
+ }
+ return res;
+ }
+
+ public void test01() {
+ tstInts(new int[] {});
+ }
+ public void test02() {
+ tstInts(new int[] {0});
+ }
+ public void test03() {
+ tstInts(new int[] {0,Integer.MAX_VALUE});
+ }
+ public void test04a() {
+ tstInts(new int[] {0, VB2 - 1});
+ }
+ public void test04b() {
+ tstInts(new int[] {0, VB2});
+ }
+ public void test04c() {
+ tstInts(new int[] {0, VB2 + 1});
+ }
+ public void test05() {
+ tstInts(fibArray(0,1,7)); // includes duplicate value 1
+ }
+ public void test05b() {
+ tstInts(reverseDiffs(fibArray(0,1,7)));
+ }
+ public void test06() {
+ tstInts(fibArray(1,2,45)); // no duplicates, size 46 exceeds max int.
+ }
+ public void test06b() {
+ tstInts(reverseDiffs(fibArray(1,2,45)));
+ }
+ public void test07a() {
+ tstInts(new int[] {0, VB3});
+ }
+ public void test07b() {
+ tstInts(new int[] {1, VB3 + 2});
+ }
+ public void test07c() {
+ tstInts(new int[] {2, VB3 + 4});
+ }
+ public void test08a() {
+ tstInts(new int[] {0, VB4 + 1});
+ }
+ public void test08b() {
+ tstInts(new int[] {1, VB4 + 1});
+ }
+ public void test08c() {
+ tstInts(new int[] {2, VB4 + 1});
+ }
+
+ public void test10() {
+ tstIllegalArgExc(new int[] {-1});
+ }
+ public void test11() {
+ tstIllegalArgExc(new int[] {1,0});
+ }
+ public void test12() {
+ tstIllegalArgExc(new int[] {0,1,1,2,3,5,8,0});
+ }
+}
Property changes on: src/test/org/apache/lucene/util/TestSortedVIntList.java
___________________________________________________________________
Name: svn:eol-style
+ native