Index: src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 880933) +++ src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -21,12 +21,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; -import java.util.Map; -import java.util.HashMap; import java.util.Set; import org.apache.lucene.analysis.KeywordAnalyzer; @@ -40,12 +40,12 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.MockRAMDirectory; -import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.BitVector; +import org.apache.lucene.util.PagedBitVector; public class TestIndexReaderReopen extends LuceneTestCase { @@ -1175,7 +1175,7 @@ // At this point they share the same BitVector assertTrue(sr1.deletedDocs==sr2.deletedDocs); - final BitVector delDocs = sr1.deletedDocs; + final PagedBitVector delDocs = sr1.deletedDocs; r1.close(); r2.deleteDocument(0); Index: src/test/org/apache/lucene/util/TestPagedBitVector.java =================================================================== --- src/test/org/apache/lucene/util/TestPagedBitVector.java (revision 0) +++ src/test/org/apache/lucene/util/TestPagedBitVector.java (revision 0) @@ -0,0 +1,176 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; + +import org.apache.lucene.store.MockRAMDirectory; + +public class TestPagedBitVector extends LuceneTestCase { + Random random = new Random(System.currentTimeMillis()); + + public void testClear() throws Exception { + PagedBitVector pbv = new PagedBitVector(1000); + pbv.set(400); + assertTrue(pbv.get(400)); + pbv.clear(400); + assertFalse(pbv.get(400)); + } + + public void testSaveLoad() throws Exception { + MockRAMDirectory dir = new MockRAMDirectory(); + int size = random.nextInt(1000*1000); + PagedBitVector bv = new PagedBitVector(size); + for (int x=0; x < size; x++) { + boolean d = random.nextInt(5) == 1; + if (d) bv.set(x); + } + bv.write(dir, "bv.del"); + PagedBitVector bv2 = new PagedBitVector(dir, "bv.del"); + verify(bv, bv2); + dir.close(); + } + + // randomly delete, save the BV + // open BV, clone, etc + public void test() throws Exception { + long duration = 1000 * 60; + long startTime = System.currentTimeMillis(); + MockRAMDirectory dir = new MockRAMDirectory(); + int iters = 0; + while (true) { + int size = random.nextInt(1000 * 1000 * 20); + BVWrapper bvw = init(size); + verify(bvw); + randomUpdates(bvw); + verify(bvw); + BVWrapper bvw2 = (BVWrapper)bvw.clone(); + assertEquals(bvw.bv.count(), bvw2.bv.count()); + randomUpdates(bvw2); + verify(bvw2); + verify(bvw); + // save, load, verify + String fileName = iters+".del"; + String fileName2 = iters+"-2.del"; + + if (random.nextInt(10) == 5) { + bvw.bv.write(dir, fileName); + bvw2.bv.write(dir, fileName2); + + PagedBitVector loadedBV = new PagedBitVector(dir, fileName); + verify(bvw.bv, loadedBV); + PagedBitVector loadedBV2 = new PagedBitVector(dir, fileName2); + verify(bvw2.bv, loadedBV2); + dir.deleteFile(fileName); + dir.deleteFile(fileName2); + } + + if ( (System.currentTimeMillis() - startTime) > duration) { + break; + } + //if (random.nextBoolean()) { + // clone the bv + + //} + iters++; + } + dir.close(); + } + + private void randomUpdates(BVWrapper bvw) throws IOException { + int size = bvw.bv.size(); + for (int x=0; x < size; x++) { + int i = random.nextInt(10); + if (i == 0) { + //bvw.clear(x); + } else if (i == 1) { + if (!bvw.bv.get(x)) bvw.set(x); + } + } + } + + public void verify(PagedBitVector bv1, PagedBitVector bv2) { + assertEquals(bv1.size(), bv2.size()); + //assertEquals(bv1.count(), bv2.count()); + int size = bv1.size(); + for (int x=0; x < size; x++) { + assertEquals("doc:"+x, bv1.get(x), bv2.get(x)); + } + } + + public void verify(BVWrapper bvw) { + for (int x=0; x < bvw.actual.length; x++) { + assertEquals(bvw.actual[x], bvw.bv.get(x)); + } + assertEquals(bvw.countBools(), bvw.bv.count()); + } + + private BVWrapper init(int size) { + BVWrapper bv = new BVWrapper(); + bv.bv = new PagedBitVector(size); + bv.actual = new boolean[size]; + Arrays.fill(bv.actual, false); + return bv; + } + + private void fill(BVWrapper bv) throws IOException { + int size = bv.bv.size(); + for (int x=0; x < size; x++) { + if (random.nextBoolean()) { + bv.set(x); + } + } + } + + public static class BVWrapper implements Cloneable { + PagedBitVector bv; + boolean[] actual; + + public Object clone() { + BVWrapper clone = new BVWrapper(); + clone.bv = (PagedBitVector)bv.clone(); + clone.actual = new boolean[actual.length]; + for (int x=0; x < actual.length; x++) { + clone.actual[x] = actual[x]; + } + return clone; + } + + public int countBools() { + int count = 0; + for (int x=0; x < actual.length; x++) { + if (actual[x]) { + count++; + } + } + return count; + } + + public void clear(int doc) throws IOException { + bv.clear(doc); + actual[doc] = false; + } + + public void set(int doc) throws IOException { + bv.set(doc); + actual[doc] = true; + } + } +} Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 880933) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; - import java.util.List; import java.util.Map; import java.util.Set; @@ -37,6 +36,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BitVector; import org.apache.lucene.util.CloseableThreadLocal; +import org.apache.lucene.util.PagedBitVector; /** @version $Id */ /** @@ -52,7 +52,7 @@ CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); - BitVector deletedDocs = null; + PagedBitVector deletedDocs = null; Ref deletedDocsRef = null; private boolean deletedDocsDirty = false; private boolean normsDirty = false; @@ -601,7 +601,7 @@ private void loadDeletedDocs() throws IOException { // NOTE: the bitvector is stored using the regular directory, not cfs if (hasDeletions(si)) { - deletedDocs = new BitVector(directory(), si.getDelFileName()); + deletedDocs = new PagedBitVector(directory(), si.getDelFileName()); deletedDocsRef = new Ref(); assert si.getDelCount() == deletedDocs.count() : @@ -632,8 +632,8 @@ * @param bv BitVector to clone * @return New BitVector */ - protected BitVector cloneDeletedDocs(BitVector bv) { - return (BitVector)bv.clone(); + protected PagedBitVector cloneDeletedDocs(PagedBitVector bv) { + return (PagedBitVector)bv.clone(); } @Override @@ -819,7 +819,7 @@ @Override protected void doDelete(int docNum) { if (deletedDocs == null) { - deletedDocs = new BitVector(maxDoc()); + deletedDocs = new PagedBitVector(maxDoc()); deletedDocsRef = new Ref(); } // there is more than 1 SegmentReader with a reference to this Index: src/java/org/apache/lucene/index/SegmentTermDocs.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermDocs.java (revision 880933) +++ src/java/org/apache/lucene/index/SegmentTermDocs.java (working copy) @@ -18,15 +18,18 @@ */ import java.io.IOException; -import org.apache.lucene.util.BitVector; + import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.PagedBitVector; class SegmentTermDocs implements TermDocs { protected SegmentReader parent; protected IndexInput freqStream; protected int count; protected int df; - protected BitVector deletedDocs; + protected PagedBitVector deletedDocs; + private byte[] pageBytes; + private int page = -1; int doc = 0; int freq; @@ -123,9 +126,22 @@ } count++; - - if (deletedDocs == null || !deletedDocs.get(doc)) + if (deletedDocs == null) { break; + } + if (page != (doc >> PagedBitVector.CONST)) { + page = doc >> PagedBitVector.CONST; + pageBytes = deletedDocs.getPageBytes(page); + if (pageBytes == null) { + pageBytes = PagedBitVector.EMPTY_BYTES; + } + } + if ((pageBytes[(doc & PagedBitVector.MASK) >> 3] & (1 << ((doc & PagedBitVector.MASK) & 7))) == 0) { + break; + } + + //if (deletedDocs == null || !deletedDocs.get(doc)) + // break; skippingDoc(); } return true; @@ -149,10 +165,23 @@ freq = freqStream.readVInt(); // else read freq count++; - if (deletedDocs == null || !deletedDocs.get(doc)) { + if (deletedDocs == null) { docs[i] = doc; freqs[i] = freq; ++i; + } else { + if (page != (doc >> PagedBitVector.CONST)) { + page = doc >> PagedBitVector.CONST; + pageBytes = deletedDocs.getPageBytes(page); + if (pageBytes == null) { + pageBytes = PagedBitVector.EMPTY_BYTES; + } + } + if ((pageBytes[(doc & PagedBitVector.MASK) >> 3] & (1 << ((doc & PagedBitVector.MASK) & 7))) == 0) { + docs[i] = doc; + freqs[i] = freq; + ++i; + } } } return i; @@ -165,7 +194,27 @@ // manually inlined call to next() for speed doc += freqStream.readVInt(); count++; - + + if (deletedDocs == null) { + docs[i] = doc; + freqs[i] = 1; + ++i; + } else { + if (page != (doc >> PagedBitVector.CONST)) { + page = doc >> PagedBitVector.CONST; + pageBytes = deletedDocs.getPageBytes(page); + if (pageBytes == null) { + pageBytes = PagedBitVector.EMPTY_BYTES; + } + } + if ((pageBytes[(doc & PagedBitVector.MASK) >> 3] & (1 << ((doc & PagedBitVector.MASK) & 7))) == 0) { + docs[i] = doc; + freqs[i] = 1; + ++i; + } + } + + /** if (deletedDocs == null || !deletedDocs.get(doc)) { docs[i] = doc; // Hardware freq to 1 when term freqs were not @@ -173,6 +222,7 @@ freqs[i] = 1; ++i; } + **/ } return i; } Index: src/java/org/apache/lucene/index/AllTermDocs.java =================================================================== --- src/java/org/apache/lucene/index/AllTermDocs.java (revision 880933) +++ src/java/org/apache/lucene/index/AllTermDocs.java (working copy) @@ -17,13 +17,16 @@ package org.apache.lucene.index; -import org.apache.lucene.util.BitVector; import java.io.IOException; +import org.apache.lucene.util.PagedBitVector; + class AllTermDocs implements TermDocs { - protected BitVector deletedDocs; + protected PagedBitVector deletedDocs; protected int maxDoc; protected int doc = -1; + private int page = -1; + private byte[] pageBytes; protected AllTermDocs(SegmentReader parent) { synchronized (parent) { @@ -60,11 +63,31 @@ final int length = docs.length; int i = 0; while (i < length && doc < maxDoc) { + if (deletedDocs == null) { + docs[i] = doc; + freqs[i] = 1; + ++i; + } else { + if (page != (doc >> PagedBitVector.CONST)) { + page = doc >> PagedBitVector.CONST; + pageBytes = deletedDocs.getPageBytes(page); + if (pageBytes == null) { + pageBytes = PagedBitVector.EMPTY_BYTES; + } + } + if ((pageBytes[(doc & PagedBitVector.MASK) >> 3] & (1 << ((doc & PagedBitVector.MASK) & 7))) == 0) { + docs[i] = doc; + freqs[i] = 1; + ++i; + } + } + /** if (deletedDocs == null || !deletedDocs.get(doc)) { docs[i] = doc; freqs[i] = 1; ++i; } + **/ doc++; } return i; @@ -73,9 +96,24 @@ public boolean skipTo(int target) throws IOException { doc = target; while (doc < maxDoc) { + if (deletedDocs == null) { + return true; + } + if (page != (doc >> PagedBitVector.CONST)) { + page = doc >> PagedBitVector.CONST; + pageBytes = deletedDocs.getPageBytes(page); + if (pageBytes == null) { + pageBytes = PagedBitVector.EMPTY_BYTES; + } + } + if ((pageBytes[(doc & PagedBitVector.MASK) >> 3] & (1 << ((doc & PagedBitVector.MASK) & 7))) == 0) { + return true; + } + /** if (deletedDocs == null || !deletedDocs.get(doc)) { return true; } + **/ doc++; } return false; Index: src/java/org/apache/lucene/util/PagedBitVector.java =================================================================== --- src/java/org/apache/lucene/util/PagedBitVector.java (revision 0) +++ src/java/org/apache/lucene/util/PagedBitVector.java (revision 0) @@ -0,0 +1,395 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +public final class PagedBitVector implements Cloneable { + public static final int CONST = 17; + public static int MASK = (1 << CONST) - 1; + public static int BYTESLENGTH = (MASK >> 3) + 1; + public static final byte[] EMPTY_BYTES = new byte[BYTESLENGTH]; + private byte[][] pages; + private int[] counts; + private boolean[] copy; + private int size; + + /** Constructs a paged vector capable of holding n bits. */ + public PagedBitVector(int size) { + this.size = size; + int numpages = getNumPages(size); + pages = new byte[numpages][]; + counts = new int[numpages]; + Arrays.fill(counts, -1); + copy = new boolean[numpages]; + Arrays.fill(copy, false); + } + + PagedBitVector(byte[][] pages, int size, int[] counts, boolean[] copy) { + this.pages = pages; + this.size = size; + this.counts = counts; + this.copy = copy; + } + + public final byte[] getPageBytes(int page) { + return pages[page]; + } + + /** + * The maximum number of documents this BitVector + * can hold. + */ + public final int getMaxDoc() { + return (MASK+1) * pages.length; + } + + public static final int getPageMaxDoc(int page) { + return page * (MASK + 1); + } + + public static final int getNumPages(int size) { + return (size >> CONST) + 1; + } + + public static final int getPageBytePos(int doc) { + int ndocid = ((doc & MASK)); + return ndocid >> 3; + } + + public static final int getDocInPage(int doc) { + return doc & MASK; + } + + public static final int getByteInPage(int doc) { + return (doc & MASK) >> 3; + } + + public static final int getPage(int doc) { + return doc >> CONST; + } + + public final int count() { + int total = 0; + //for (int c : counts) { + assert counts.length == pages.length; + for (int x=0; x < pages.length; x++) { + total += count(x); + } + return total; + } + + @Override + public Object clone() { + byte[][] pagesCopy = new byte[pages.length][]; + int[] countsCopy = new int[pages.length]; + boolean[] copy2 = new boolean[pages.length]; + for (int x=0; x < pages.length; x++) { + countsCopy[x] = counts[x]; + if (pages[x] != null) { + copy2[x] = true; + pagesCopy[x] = pages[x]; + } + } + return new PagedBitVector(pagesCopy, size, countsCopy, copy2); + } + + public final void set(int doc) { + int page = doc >> CONST; + if (page > pages.length-1) { + throw new ArrayIndexOutOfBoundsException("page > "+pages.length); + } + int pb = (doc & MASK) >> 3; + int pdoc = doc & MASK; + // if the page is a reference, we need to copy it + makeWriteable(page); + pages[page][pb] |= 1 << (pdoc & 7); + counts[page] = -1; + } + + // make sure the page is writeable + // create the page if it's null + // + private void makeWriteable(int page) { + if (pages[page] == null) { + pages[page] = new byte[BYTESLENGTH]; + assert !copy[page]; + } else if (copy[page]) { + makeCopy(page); + } + } + + /** Sets the value of bit to zero. */ + public final void clear(int doc) { + int page = doc >> CONST; + int pb = (doc & MASK) >> 3; + int pdoc = doc & MASK; + makeWriteable(page); + pages[page][pb] &= ~(1 << (pdoc & 7)); + counts[page] = -1; + } + + private final void makeCopy(int page) { + assert copy[page]; + final byte[] pageBytes = pages[page]; + final byte[] pageBytesCopy = new byte[pageBytes.length]; + System.arraycopy(pageBytes, 0, pageBytesCopy, 0, pageBytes.length); + pages[page] = pageBytesCopy; + // the page is not a reference anymore + copy[page] = false; + } + + public final boolean getAndSet(int doc) { + boolean b = get(doc); + set(doc); + return b; + } + + /** Returns true if bit is one and + false if it is zero. */ + public final boolean get(int doc) { + int page = doc >> CONST; + int pb = (doc & MASK) >> 3; + int pdoc = doc & MASK; + if (pages[page] == null) { + return false; + } + byte[] pageBytes = pages[page]; + return (pageBytes[pb] & (1 << (pdoc & 7))) != 0; + } + + public static final boolean get(int doc, byte[] pageBytes) { + int pdoc = doc & MASK; + int pb = (doc & MASK) >> 3; + return (pageBytes[pb] & (1 << (pdoc & 7))) != 0; + } + + /** Returns the number of bits in this vector. This is also one greater than + the number of the largest valid bit number. */ + public final int size() { + return size; + } + + public final int size(int page) { + return MASK+1; + } + + /** Returns the total number of one bits in a page. This is efficiently + computed and cached, so that, if the vector is not changed, no + recomputation is done for repeated calls. */ + public final int count(int page) { + int count = counts[page]; + if (count == -1) { + int c = 0; + byte[] pageBytes = pages[page]; + if (pageBytes != null) { + int end = pageBytes.length; + for (int i = 0; i < end; i++) + c += BYTE_COUNTS[pageBytes[i] & 0xFF]; // sum bits per byte + count = c; + counts[page] = count; + } else { + count = 0; + } + } + return count; + } + + private static final byte[] BYTE_COUNTS = { // table of bits/byte + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 + }; + + /** Constructs a bit vector from the file name in Directory + d, as written by the {@link #write} method. + */ +public PagedBitVector(Directory d, String name) throws IOException { + IndexInput input = d.openInput(name); + try { + size = input.readInt(); // read size + int numPages = input.readInt(); + int bytesLen = input.readInt(); + assert bytesLen == BYTESLENGTH; + + counts = new int[numPages]; + copy = new boolean[numPages]; + Arrays.fill(copy, false); + pages = new byte[numPages][]; + for (int x=0; x < numPages; x++) { + int page = input.readInt(); + assert page == x : "page:"+x+" read:"+page; + int size = input.readInt(); + int count = input.readInt(); + counts[x] = count; + int type = input.readInt(); + if (type == 0) { + // skip there's nothing for this page + counts[x] = 0; + } else if (type == 1) { + readDgaps(input, x, size, count); + } else if (type == 2) { + readBytes(input, x, size, count); + } + } + } finally { + input.close(); + } +} + + /** Writes this vector to the file name in Directory + d, in a format that can be read by the constructor {@link + #BitVector(Directory, String)}. */ + public final void write(Directory d, String name) throws IOException { + IndexOutput output = d.createOutput(name); + output.writeInt(size); + output.writeInt(pages.length); + output.writeInt(BYTESLENGTH); + try { + for (int x=0; x < pages.length; x++) { + output.writeInt(x); // write page + output.writeInt(size(x)); + output.writeInt(count(x)); + if (pages[x] == null) { + output.writeInt(0); + } else if (isSparse(x)) { + output.writeInt(1); + writeDgaps(output, x); // sparse bit-set more efficiently saved as d-gaps. + } else { + output.writeInt(2); + writeBytes(output, x); + } + } + } finally { + output.close(); + } + } + + /** Write as a bit set */ + private void writeBytes(IndexOutput output, int page) throws IOException { + byte[] pageBytes = pages[page]; + output.writeInt(pageBytes.length); + output.writeBytes(pageBytes, pageBytes.length); + } + + /** Indicates if the bit vector is sparse and should be saved as a d-gaps list, or dense, and should be saved as a bit set. */ + private boolean isSparse(int page) { + byte[] pageBytes = pages[page]; + // note: order of comparisons below set to favor smaller values (no binary range search.) + // note: adding 4 because we start with ((int) -1) to indicate d-gaps format. + // note: we write the d-gap for the byte number, and the byte (bits[i]) itself, therefore + // multiplying count by (8+8) or (8+16) or (8+24) etc.: + // - first 8 for writing bits[i] (1 byte vs. 1 bit), and + // - second part for writing the byte-number d-gap as vint. + // note: factor is for read/write of byte-arrays being faster than vints. + int count = count(page); + int size = size(page); + int factor = 10; + if (pageBytes.length < (1<< 7)) return factor * (4 + (8+ 8)*count) < size; + if (pageBytes.length < (1<<14)) return factor * (4 + (8+16)*count) < size; + if (pageBytes.length < (1<<21)) return factor * (4 + (8+24)*count) < size; + if (pageBytes.length < (1<<28)) return factor * (4 + (8+32)*count) < size; + return factor * (4 + (8+40)*count) < size; + } + + + + /** Read as a bit set */ + private void readBytes(IndexInput input, int page, int size, int count) throws IOException { + byte[] pageBytes = new byte[BYTESLENGTH]; + int len = input.readInt(); + assert len == pageBytes.length; + input.readBytes(pageBytes, 0, pageBytes.length); + pages[page] = pageBytes; + } + + /** read as a d-gaps list */ + private void readDgaps(IndexInput input, int page, int size, int count) throws IOException { + counts[page] = count; + byte[] pageBytes = new byte[BYTESLENGTH]; + int last=0; + int n = count; + while (n>0) { + last += input.readVInt(); + pageBytes[last] = input.readByte(); + n -= BYTE_COUNTS[pageBytes[last] & 0xFF]; + } + pages[page] = pageBytes; + } + + /** Write as a d-gaps list */ + private void writeDgaps(IndexOutput output, int page) throws IOException { + byte[] pageBytes = pages[page]; + int last=0; + int n = count(page); + int m = pageBytes.length; + for (int i=0; i0; i++) { + if (pageBytes[i]!=0) { + output.writeVInt(i-last); + output.writeByte(pageBytes[i]); + last = i; + n -= BYTE_COUNTS[pageBytes[i] & 0xFF]; + } + } + } + /** + * Retrieve a subset of this BitVector. + * + * @param start + * starting index, inclusive + * @param end + * ending index, exclusive + * @return subset + */ + /** + public PagedBitVector subset(int start, int end) { + if (start < 0 || end > size() || end < start) + throw new IndexOutOfBoundsException(); + // Special case -- return empty vector is start == end + if (end == start) return new PagedBitVector(0); + byte[] bits = new byte[((end - start - 1) >>> 3) + 1]; + int s = start >>> 3; + for (int i = 0; i < bits.length; i++) { + int cur = 0xFF & this.bits[i + s]; + int next = i + s + 1 >= this.bits.length ? 0 : 0xFF & this.bits[i + s + 1]; + bits[i] = (byte) ((cur >>> (start & 7)) | ((next << (8 - (start & 7))))); + } + int bitsToClear = (bits.length * 8 - (end - start)) % 8; + bits[bits.length - 1] &= ~(0xFF << (8 - bitsToClear)); + return new PagedBitVector(bits, end - start); + } + **/ +}