Index: contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java =================================================================== --- contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java (revision 822088) +++ contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java (working copy) @@ -36,6 +36,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermRef; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -212,8 +213,8 @@ public void setNextReader(IndexReader reader, int docBase) throws IOException { docValues = FieldCache.DEFAULT.getInts(reader, "parser", new FieldCache.IntParser() { - public final int parseInt(final String val) { - return (val.charAt(0)-'A') * 123456; + public final int parseInt(TermRef termRef) { + return (termRef.toString().charAt(0)-'A') * 123456; } }); } Index: src/java/org/apache/lucene/index/AllDocsEnum.java =================================================================== --- src/java/org/apache/lucene/index/AllDocsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/AllDocsEnum.java (revision 0) @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import org.apache.lucene.util.Bits; +import java.io.IOException; + +class AllDocsEnum extends DocsEnum { + protected final Bits skipDocs; + protected final int maxDoc; + protected final IndexReader reader; + protected int doc = -1; + + protected AllDocsEnum(IndexReader reader, Bits skipDocs) { + this.skipDocs = skipDocs; + this.maxDoc = reader.maxDoc(); + this.reader = reader; + } + + public int freq() { + return 1; + } + + public int next() throws IOException { + return advance(doc+1); + } + + public int read(int[] docs, int[] freqs) throws IOException { + final int length = docs.length; + int i = 0; + while (i < length && doc < maxDoc) { + if (skipDocs == null || !skipDocs.get(doc)) { + docs[i] = doc; + freqs[i] = 1; + ++i; + } + doc++; + } + return i; + } + + public int advance(int target) throws IOException { + doc = target; + while (doc < maxDoc) { + if (skipDocs == null || !skipDocs.get(doc)) { + return doc; + } + doc++; + } + doc = NO_MORE_DOCS; + return doc; + } + + public PositionsEnum positions() { + throw new UnsupportedOperationException(); + } +} Index: src/java/org/apache/lucene/index/AllTermDocs.java =================================================================== --- src/java/org/apache/lucene/index/AllTermDocs.java (revision 822088) +++ src/java/org/apache/lucene/index/AllTermDocs.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.util.BitVector; import java.io.IOException; +/** @deprecated Switch to AllDocsEnum */ class AllTermDocs implements TermDocs { protected BitVector deletedDocs; protected int maxDoc; Index: src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- src/java/org/apache/lucene/index/CheckIndex.java (revision 822088) +++ src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -22,6 +22,8 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.document.AbstractField; // for javadocs import org.apache.lucene.document.Document; +import org.apache.lucene.index.codecs.Codecs; +import org.apache.lucene.util.Bits; import java.text.NumberFormat; import java.io.PrintStream; @@ -271,24 +273,6 @@ infoStream.println(msg); } - private static class MySegmentTermDocs extends SegmentTermDocs { - - int delCount; - - MySegmentTermDocs(SegmentReader p) { - super(p); - } - - public void seek(Term term) throws IOException { - super.seek(term); - delCount = 0; - } - - protected void skippingDoc() throws IOException { - delCount++; - } - } - /** Returns true if index is clean, else false. * @deprecated Please instantiate a CheckIndex and then use {@link #checkIndex()} instead */ public static boolean check(Directory dir, boolean doFix) throws IOException { @@ -319,6 +303,10 @@ return checkIndex(null); } + protected Status checkIndex(List onlySegments) throws IOException { + return checkIndex(onlySegments, Codecs.getDefault()); + } + /** Returns a {@link Status} instance detailing * the state of the index. * @@ -331,13 +319,13 @@ *

WARNING: make sure * you only call this when the index is not opened by any * writer. */ - public Status checkIndex(List onlySegments) throws IOException { + protected Status checkIndex(List onlySegments, Codecs codecs) throws IOException { NumberFormat nf = NumberFormat.getInstance(); SegmentInfos sis = new SegmentInfos(); Status result = new Status(); result.dir = dir; try { - sis.read(dir); + sis.read(dir, codecs); } catch (Throwable t) { msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; @@ -394,6 +382,8 @@ sFormat = "FORMAT_USER_DATA [Lucene 2.9]"; else if (format == SegmentInfos.FORMAT_DIAGNOSTICS) sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]"; + else if (format == SegmentInfos.FORMAT_FLEX_POSTINGS) + sFormat = "FORMAT_FLEX_POSTINGS [Lucene 2.9]"; else if (format < SegmentInfos.CURRENT_FORMAT) { sFormat = "int=" + format + " [newer version of Lucene than this tool]"; skip = true; @@ -615,66 +605,87 @@ private Status.TermIndexStatus testTermIndex(SegmentInfo info, SegmentReader reader) { final Status.TermIndexStatus status = new Status.TermIndexStatus(); + final int maxDoc = reader.maxDoc(); + final Bits delDocs = reader.getDeletedDocs(); + try { + if (infoStream != null) { infoStream.print(" test: terms, freq, prox..."); } + + final FieldsEnum fields = reader.fields().iterator(); + while(true) { + final String field = fields.next(); + if (field == null) { + break; + } + + final TermsEnum terms = fields.terms(); + while(true) { - final TermEnum termEnum = reader.terms(); - final TermPositions termPositions = reader.termPositions(); + final TermRef term = terms.next(); + if (term == null) { + break; + } + final int docFreq = terms.docFreq(); + status.totFreq += docFreq; - // Used only to count up # deleted docs for this term - final MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader); + final DocsEnum docs = terms.docs(delDocs); + status.termCount++; - final int maxDoc = reader.maxDoc(); + int lastDoc = -1; + int freq0 = 0; + while(true) { + final int doc = docs.next(); + if (doc == DocsEnum.NO_MORE_DOCS) { + break; + } + final int freq = docs.freq(); + status.totPos += freq; - while (termEnum.next()) { - status.termCount++; - final Term term = termEnum.term(); - final int docFreq = termEnum.docFreq(); - termPositions.seek(term); - int lastDoc = -1; - int freq0 = 0; - status.totFreq += docFreq; - while (termPositions.next()) { - freq0++; - final int doc = termPositions.doc(); - final int freq = termPositions.freq(); - if (doc <= lastDoc) - throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc); - if (doc >= maxDoc) - throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc); + freq0++; + if (doc <= lastDoc) { + throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc); + } + if (doc >= maxDoc) { + throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc); + } - lastDoc = doc; - if (freq <= 0) - throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds"); + lastDoc = doc; + if (freq <= 0) { + throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds"); + } - int lastPos = -1; - status.totPos += freq; - for(int j=0;j>>= 1; - } else { - delta = skipStream.readVInt(); - } - freqPointer[level] += skipStream.readVInt(); - proxPointer[level] += skipStream.readVInt(); - - return delta; - } -} Index: src/java/org/apache/lucene/index/DefaultSkipListWriter.java =================================================================== --- src/java/org/apache/lucene/index/DefaultSkipListWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/DefaultSkipListWriter.java (working copy) @@ -1,134 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.store.IndexOutput; - - -/** - * Implements the skip list writer for the default posting list format - * that stores positions and payloads. - * - */ -class DefaultSkipListWriter extends MultiLevelSkipListWriter { - private int[] lastSkipDoc; - private int[] lastSkipPayloadLength; - private long[] lastSkipFreqPointer; - private long[] lastSkipProxPointer; - - private IndexOutput freqOutput; - private IndexOutput proxOutput; - - private int curDoc; - private boolean curStorePayloads; - private int curPayloadLength; - private long curFreqPointer; - private long curProxPointer; - - DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput) { - super(skipInterval, numberOfSkipLevels, docCount); - this.freqOutput = freqOutput; - this.proxOutput = proxOutput; - - lastSkipDoc = new int[numberOfSkipLevels]; - lastSkipPayloadLength = new int[numberOfSkipLevels]; - lastSkipFreqPointer = new long[numberOfSkipLevels]; - lastSkipProxPointer = new long[numberOfSkipLevels]; - } - - void setFreqOutput(IndexOutput freqOutput) { - this.freqOutput = freqOutput; - } - - void setProxOutput(IndexOutput proxOutput) { - this.proxOutput = proxOutput; - } - - /** - * Sets the values for the current skip data. - */ - void setSkipData(int doc, boolean storePayloads, int payloadLength) { - this.curDoc = doc; - this.curStorePayloads = storePayloads; - this.curPayloadLength = payloadLength; - this.curFreqPointer = freqOutput.getFilePointer(); - if (proxOutput != null) - this.curProxPointer = proxOutput.getFilePointer(); - } - - protected void resetSkip() { - super.resetSkip(); - Arrays.fill(lastSkipDoc, 0); - Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list - Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); - if (proxOutput != null) - Arrays.fill(lastSkipProxPointer, proxOutput.getFilePointer()); - } - - protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { - // To efficiently store payloads in the posting lists we do not store the length of - // every payload. Instead we omit the length for a payload if the previous payload had - // the same length. - // However, in order to support skipping the payload length at every skip point must be known. - // So we use the same length encoding that we use for the posting lists for the skip data as well: - // Case 1: current field does not store payloads - // SkipDatum --> DocSkip, FreqSkip, ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // DocSkip records the document number before every SkipInterval th document in TermFreqs. - // Document numbers are represented as differences from the previous value in the sequence. - // Case 2: current field stores payloads - // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // PayloadLength --> VInt - // In this case DocSkip/2 is the difference between - // the current and the previous value. If DocSkip - // is odd, then a PayloadLength encoded as VInt follows, - // if DocSkip is even, then it is assumed that the - // current payload length equals the length at the previous - // skip point - if (curStorePayloads) { - int delta = curDoc - lastSkipDoc[level]; - if (curPayloadLength == lastSkipPayloadLength[level]) { - // the current payload length equals the length at the previous skip point, - // so we don't store the length again - skipBuffer.writeVInt(delta * 2); - } else { - // the payload length is different from the previous one. We shift the DocSkip, - // set the lowest bit and store the current payload length as VInt. - skipBuffer.writeVInt(delta * 2 + 1); - skipBuffer.writeVInt(curPayloadLength); - lastSkipPayloadLength[level] = curPayloadLength; - } - } else { - // current field does not store payloads - skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); - } - skipBuffer.writeVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); - skipBuffer.writeVInt((int) (curProxPointer - lastSkipProxPointer[level])); - - lastSkipDoc[level] = curDoc; - //System.out.println("write doc at level " + level + ": " + curDoc); - - lastSkipFreqPointer[level] = curFreqPointer; - lastSkipProxPointer[level] = curProxPointer; - } - -} Index: src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- src/java/org/apache/lucene/index/DirectoryReader.java (revision 822088) +++ src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -17,25 +17,30 @@ * limitations under the License. */ -import java.io.IOException; import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; -import java.util.Collections; -import java.util.ArrayList; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.index.codecs.Codecs; +import org.apache.lucene.util.PriorityQueue; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.ReaderUtil; /** * An IndexReader which reads indexes with multiple segments. @@ -43,6 +48,8 @@ class DirectoryReader extends IndexReader implements Cloneable { protected Directory directory; protected boolean readOnly; + + protected Codecs codecs; IndexWriter writer; @@ -63,28 +70,52 @@ private int numDocs = -1; private boolean hasDeletions = false; + private MultiFields fields; + +// static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, +// final int termInfosIndexDivisor) throws CorruptIndexException, IOException { +// return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); +// } + static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, - final int termInfosIndexDivisor) throws CorruptIndexException, IOException { + final int termInfosIndexDivisor, Codecs codecs) throws CorruptIndexException, IOException { + final Codecs codecs2; + if (codecs == null) { + codecs2 = Codecs.getDefault(); + } else { + codecs2 = codecs; + } return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { SegmentInfos infos = new SegmentInfos(); - infos.read(directory, segmentFileName); + infos.read(directory, segmentFileName, codecs2); if (readOnly) - return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor); + return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, codecs2); else - return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor); + return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, codecs2); } }.run(commit); } /** Construct reading the named set of readers. */ - DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { +// DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { +// this(directory, sis, deletionPolicy, readOnly, termInfosIndexDivisor, null); +// } + + /** Construct reading the named set of readers. */ + DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, Codecs codecs) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = sis; this.deletionPolicy = deletionPolicy; this.termInfosIndexDivisor = termInfosIndexDivisor; + if (codecs == null) { + this.codecs = Codecs.getDefault(); + } else { + this.codecs = codecs; + } + if (!readOnly) { // We assume that this segments_N was previously // properly sync'd: @@ -120,11 +151,18 @@ } // Used by near real-time search - DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException { + DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException { this.directory = writer.getDirectory(); this.readOnly = true; this.segmentInfos = infos; this.termInfosIndexDivisor = termInfosIndexDivisor; + if (codecs == null) { + this.codecs = Codecs.getDefault(); + } else { + this.codecs = codecs; + } + + if (!readOnly) { // We assume that this segments_N was previously // properly sync'd: @@ -175,11 +213,17 @@ /** This constructor is only used for {@link #reopen()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, - Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException { + Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, Codecs codecs) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = infos; this.termInfosIndexDivisor = termInfosIndexDivisor; + if (codecs == null) { + this.codecs = Codecs.getDefault(); + } else { + this.codecs = codecs; + } + if (!readOnly) { // We assume that this segments_N was previously // properly sync'd: @@ -301,14 +345,75 @@ private void initialize(SegmentReader[] subReaders) { this.subReaders = subReaders; starts = new int[subReaders.length + 1]; // build starts array + Bits[] subs = new Bits[subReaders.length]; for (int i = 0; i < subReaders.length; i++) { starts[i] = maxDoc; maxDoc += subReaders[i].maxDoc(); // compute maxDocs - if (subReaders[i].hasDeletions()) + if (subReaders[i].hasDeletions()) { hasDeletions = true; + } + subs[i] = subReaders[i].getDeletedDocs(); } starts[subReaders.length] = maxDoc; + + if (hasDeletions) { + deletedDocs = new MultiBits(subs, starts); + } else { + deletedDocs = null; + } + + fields = new MultiFields(subReaders, starts); + } + + private MultiBits deletedDocs; + + // Exposes a slice of an existing Bits as a new Bits + final static class SubBits implements Bits { + private final Bits parent; + private final int start; + private final int length; + + // start is inclusive; end is exclusive (length = end-start) + public SubBits(Bits parent, int start, int end) { + this.parent = parent; + this.start = start; + this.length = end - start; + } + + public boolean get(int doc) { + if (doc >= length) { + throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1)); + } + return parent.get(doc-start); + } + } + + // Concatenates multiple Bits together + // nocommit -- if none of the subs have deletions we + // should return null from getDeletedDocs: + static final class MultiBits implements Bits { + private final Bits[] subs; + final int[] starts; + + public MultiBits(Bits[] subs, int[] starts) { + this.subs = subs; + this.starts = starts; + } + + public boolean get(int doc) { + final int reader = ReaderUtil.subIndex(doc, starts); + final Bits bits = subs[reader]; + if (bits == null) { + return false; + } else { + return bits.get(doc-starts[reader]); + } + } + } + + public Bits getDeletedDocs() { + return deletedDocs; } public final synchronized Object clone() { @@ -423,7 +528,7 @@ return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { SegmentInfos infos = new SegmentInfos(); - infos.read(directory, segmentFileName); + infos.read(directory, segmentFileName, codecs); return doReopen(infos, false, openReadOnly); } }.run(commit); @@ -432,9 +537,9 @@ private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { DirectoryReader reader; if (openReadOnly) { - reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor); + reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, null); } else { - reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor); + reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, null); } reader.setDisableFakeNorms(getDisableFakeNorms()); return reader; @@ -626,10 +731,23 @@ return total; } + public int docFreq(String field, TermRef term) throws IOException { + ensureOpen(); + int total = 0; // sum freqs in segments + for (int i = 0; i < subReaders.length; i++) { + total += subReaders[i].docFreq(field, term); + } + return total; + } + public TermDocs termDocs() throws IOException { ensureOpen(); return new MultiTermDocs(this, subReaders, starts); } + + public Fields fields() throws IOException { + return fields; + } public TermPositions termPositions() throws IOException { ensureOpen(); @@ -669,7 +787,7 @@ // we have to check whether index has changed since this reader was opened. // if so, this reader is no longer valid for deletion - if (SegmentInfos.readCurrentVersion(directory) > segmentInfos.getVersion()) { + if (SegmentInfos.readCurrentVersion(directory, codecs) > segmentInfos.getVersion()) { stale = true; this.writeLock.release(); this.writeLock = null; @@ -699,7 +817,7 @@ // KeepOnlyLastCommitDeleter: IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, null, null); + segmentInfos, null, null, codecs); // Checkpoint the state we are about to change, in // case we have to roll back: @@ -794,7 +912,7 @@ */ public boolean isCurrent() throws CorruptIndexException, IOException { ensureOpen(); - return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion(); + return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion(); } protected synchronized void doClose() throws IOException { @@ -861,12 +979,17 @@ /** @see org.apache.lucene.index.IndexReader#listCommits */ public static Collection listCommits(Directory dir) throws IOException { + return listCommits(dir, Codecs.getDefault()); + } + + /** @see org.apache.lucene.index.IndexReader#listCommits */ + public static Collection listCommits(Directory dir, Codecs codecs) throws IOException { final String[] files = dir.listAll(); Collection commits = new ArrayList(); SegmentInfos latest = new SegmentInfos(); - latest.read(dir); + latest.read(dir, codecs); final long currentGen = latest.getGeneration(); commits.add(new ReaderCommit(latest, dir)); @@ -883,7 +1006,7 @@ try { // IOException allowed to throw there, in case // segments_N is corrupt - sis.read(dir, fileName); + sis.read(dir, fileName, codecs); } catch (FileNotFoundException fnfe) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -955,29 +1078,497 @@ } } + private final static class TermsWithBase { + Terms terms; + int base; + int length; + Bits deletedDocs; + + public TermsWithBase(IndexReader reader, int base, String field) throws IOException { + this.base = base; + length = reader.maxDoc(); + deletedDocs = reader.getDeletedDocs(); + terms = reader.fields().terms(field); + } + } + + private final static class FieldsEnumWithBase { + FieldsEnum fields; + String current; + int base; + int length; + Bits deletedDocs; + + public FieldsEnumWithBase(IndexReader reader, int base) throws IOException { + this.base = base; + length = reader.maxDoc(); + deletedDocs = reader.getDeletedDocs(); + fields = reader.fields().iterator(); + } + } + + private final static class TermsEnumWithBase { + TermsEnum terms; + int base; + int length; + TermRef current; + Bits deletedDocs; + + public TermsEnumWithBase(FieldsEnumWithBase start, TermsEnum terms, TermRef term) { + this.terms = terms; + current = term; + deletedDocs = start.deletedDocs; + base = start.base; + length = start.length; + } + + public TermsEnumWithBase(TermsWithBase start, TermsEnum terms, TermRef term) { + this.terms = terms; + current = term; + deletedDocs = start.deletedDocs; + base = start.base; + length = start.length; + } + } + + private final static class DocsEnumWithBase { + DocsEnum docs; + int base; + } + + private final static class FieldMergeQueue extends PriorityQueue { + FieldMergeQueue(int size) { + initialize(size); + } + + protected final boolean lessThan(Object a, Object b) { + FieldsEnumWithBase fieldsA = (FieldsEnumWithBase) a; + FieldsEnumWithBase fieldsB = (FieldsEnumWithBase) b; + return fieldsA.current.compareTo(fieldsB.current) < 0; + } + } + + private final static class TermMergeQueue extends PriorityQueue { + TermMergeQueue(int size) { + initialize(size); + } + + protected final boolean lessThan(Object a, Object b) { + TermsEnumWithBase termsA = (TermsEnumWithBase) a; + TermsEnumWithBase termsB = (TermsEnumWithBase) b; + final int cmp = termsA.current.compareTerm(termsB.current); + if (cmp != 0) { + return cmp < 0; + } else { + return termsA.base < termsB.base; + } + } + } + + final static class MultiFields extends Fields { + private final IndexReader[] readers; + private final int[] starts; + private final HashMap terms = new HashMap(); + + public MultiFields(IndexReader[] readers, int[] starts) { + this.readers = readers; + this.starts = starts; + } + + public FieldsEnum iterator() throws IOException { + FieldsEnumWithBase[] subs = new FieldsEnumWithBase[readers.length]; + for(int i=0;i subs = new ArrayList(); + + // Gather all sub-readers that have this field + for(int i=0;i 0) { + while(true) { + top[numTop++] = (FieldsEnumWithBase) queue.pop(); + if (queue.size() == 0 || ((FieldsEnumWithBase) queue.top()).current != top[0].current) { + break; + } + } + currentField = top[0].current; + } else { + currentField = null; + } + + return currentField; + } + + public TermsEnum terms() throws IOException { + return terms.reset(top, numTop); + } + } + + private static final class MultiTermsEnum extends TermsEnum { + + private final TermMergeQueue queue; + private final TermsEnumWithBase[] subs; + private final TermsEnumWithBase[] top; + int numTop; + int numSubs; + private TermRef current; + private final SeekResult seekResult = new SeekResult(); + private final MultiDocsEnum docs; + + MultiTermsEnum(int size) { + queue = new TermMergeQueue(size); + top = new TermsEnumWithBase[size]; + subs = new TermsEnumWithBase[size]; + docs = new MultiDocsEnum(size); + } + + MultiTermsEnum reset(TermsWithBase[] terms) throws IOException { + assert terms.length <= top.length; + numSubs = 0; + numTop = 0; + for(int i=0;i 0) { + seekResult.status = SeekResult.Status.FOUND; + } else if (queue.size() > 0) { + pullTop(); + seekResult.term = current; + seekResult.status = SeekResult.Status.NOT_FOUND; + } else { + seekResult.status = SeekResult.Status.END; + } + + return seekResult; + } + + private final void pullTop() { + assert numTop == 0; + while(true) { + top[numTop++] = (TermsEnumWithBase) queue.pop(); + if (queue.size() == 0 || !((TermsEnumWithBase) queue.top()).current.termEquals(top[0].current)) { + break; + } + } + current = top[0].current; + } + + private final void pushTop() throws IOException { + for(int i=0;i 0) { + pullTop(); + } else { + current = null; + } + + return current; + } + + public int docFreq() { + int sum = 0; + for(int i=0;i= target */ + public abstract int advance(int target) throws IOException; + + /** Returns the next docID, {@link #NO_MORE_DOCS} at the end. */ + public abstract int next() throws IOException; + + public abstract int freq(); + + // nocommit -- fix this API so that intblock codecs are + // able to return their own int arrays, to save a copy + /** Bulk read: returns number of docs read. Subclass may + * do this more efficiently. */ + public int read(int[] docs, int[] freqs) throws IOException { + int count = 0; + while(count < docs.length) { + final int doc = next(); + if (doc != NO_MORE_DOCS) { + docs[count] = doc; + freqs[count] = freq(); + count++; + } else { + break; + } + } + return count; + } + + // nocommit -- maybe move this up to TermsEnum? that + // would disallow changing positions format/reader of each + // doc, though + // nocommit - doc whether this returns null if there are + // no positions, or a faker + /** Don't call next() or skipTo() or read() until you're + * done consuming the positions */ + public abstract PositionsEnum positions() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/DocsEnum.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -40,6 +40,8 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Constants; +import org.apache.lucene.index.codecs.Codec; + /** * This class accepts multiple added documents and directly * writes a single segment file. It does this more @@ -543,9 +545,16 @@ synchronized private void initFlushState(boolean onlyDocStore) { initSegmentName(onlyDocStore); - flushState = new SegmentWriteState(this, directory, segment, docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval()); + flushState = new SegmentWriteState(this, directory, segment, docFieldProcessor.fieldInfos, + docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval(), + writer.codecs); } + /** Returns the codec used to flush the last segment */ + Codec getCodec() { + return flushState.codec; + } + /** Flush all pending docs to a new segment */ synchronized int flush(boolean closeDocStore) throws IOException { @@ -581,7 +590,8 @@ consumer.flush(threads, flushState); if (infoStream != null) { - final long newSegmentSize = segmentSize(flushState.segmentName); + SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, flushState.codec); + final long newSegmentSize = si.sizeInBytes(); String message = " oldRAMSize=" + numBytesUsed + " newFlushedSize=" + newSegmentSize + " docs/MB=" + nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) + @@ -611,8 +621,12 @@ CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION); Iterator it = flushState.flushedFiles.iterator(); - while(it.hasNext()) - cfsWriter.addFile((String) it.next()); + while(it.hasNext()) { + final String fileName = (String) it.next(); + if (Codec.DEBUG) + System.out.println("make cfs " + fileName); + cfsWriter.addFile(fileName); + } // Perform the merge cfsWriter.close(); @@ -968,24 +982,27 @@ // Delete by term Iterator iter = deletesFlushed.terms.entrySet().iterator(); - TermDocs docs = reader.termDocs(); + try { while (iter.hasNext()) { Entry entry = (Entry) iter.next(); Term term = (Term) entry.getKey(); - docs.seek(term); - int limit = ((BufferedDeletes.Num) entry.getValue()).getNum(); - while (docs.next()) { - int docID = docs.doc(); - if (docIDStart+docID >= limit) - break; - reader.deleteDocument(docID); - any = true; + DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field, new TermRef(term.text)); + if (docs != null) { + int limit = ((BufferedDeletes.Num) entry.getValue()).getNum(); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) { + break; + } + reader.deleteDocument(docID); + any = true; + } } } } finally { - docs.close(); + //docs.close(); } // Delete by docID @@ -1138,24 +1155,6 @@ NumberFormat nf = NumberFormat.getInstance(); - // TODO FI: this is not flexible -- we can't hardwire - // extensions in here: - private long segmentSize(String segmentName) throws IOException { - // Used only when infoStream != null - assert infoStream != null; - - long size = directory.fileLength(segmentName + ".tii") + - directory.fileLength(segmentName + ".tis") + - directory.fileLength(segmentName + ".frq") + - directory.fileLength(segmentName + ".prx"); - - final String normFileName = segmentName + ".nrm"; - if (directory.fileExists(normFileName)) - size += directory.fileLength(normFileName); - - return size; - } - // Coarse estimates used to measure RAM usage of buffered deletes final static int OBJECT_HEADER_BYTES = 8; final static int POINTER_NUM_BYTE = Constants.JRE_IS_64BIT ? 8 : 4; Index: src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- src/java/org/apache/lucene/index/FieldInfo.java (revision 822088) +++ src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -17,20 +17,27 @@ * limitations under the License. */ -final class FieldInfo { - String name; - boolean isIndexed; - int number; +// nocommit -- made this public: +public final class FieldInfo { + // nocommit -- made this public + public String name; + // nocommit -- made this public + public boolean isIndexed; + // nocommit -- made this public + public int number; // true if term vector for this field should be stored boolean storeTermVector; boolean storeOffsetWithTermVector; boolean storePositionWithTermVector; - boolean omitNorms; // omit norms associated with indexed fields - boolean omitTermFreqAndPositions; - - boolean storePayloads; // whether this field stores payloads together with term positions + // nocommit -- made this public + public boolean omitNorms; // omit norms associated with indexed fields + // nocommit -- made this public + public boolean omitTermFreqAndPositions; + + // nocommit -- made public + public boolean storePayloads; // whether this field stores payloads together with term positions FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, Index: src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- src/java/org/apache/lucene/index/FieldInfos.java (revision 822088) +++ src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -33,7 +33,8 @@ * be adding documents at a time, with no other reader or writer threads * accessing this object. */ -final class FieldInfos { +// nocommit -- made this public: +public final class FieldInfos { // Used internally (ie not written to *.fnm files) for pre-2.9 files public static final int FORMAT_PRE = -1; @@ -121,14 +122,19 @@ } /** Returns true if any fields do not omitTermFreqAndPositions */ - boolean hasProx() { + // nocommit -- made public + public boolean hasProx() { final int numFields = byNumber.size(); for(int i=0;i 0 && delta <= 0)) - throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); - - if ((++df % skipInterval) == 0) { - // TODO: abstraction violation - skipListWriter.setSkipData(lastDocID, storePayloads, posWriter.lastPayloadLength); - skipListWriter.bufferSkip(df); - } - - assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; - - lastDocID = docID; - if (omitTermFreqAndPositions) - out.writeVInt(delta); - else if (1 == termDocFreq) - out.writeVInt((delta<<1) | 1); - else { - out.writeVInt(delta<<1); - out.writeVInt(termDocFreq); - } - - return posWriter; - } - - private final TermInfo termInfo = new TermInfo(); // minimize consing - final UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result(); - - /** Called when we are done adding docs to this term */ - void finish() throws IOException { - long skipPointer = skipListWriter.writeSkip(out); - - // TODO: this is abstraction violation -- we should not - // peek up into parents terms encoding format - termInfo.set(df, parent.freqStart, parent.proxStart, (int) (skipPointer - parent.freqStart)); - - // TODO: we could do this incrementally - UnicodeUtil.UTF16toUTF8(parent.currentTerm, parent.currentTermStart, utf8); - - if (df > 0) { - parent.termsOut.add(fieldInfo.number, - utf8.result, - utf8.length, - termInfo); - } - - lastDocID = 0; - df = 0; - } - - void close() throws IOException { - out.close(); - posWriter.close(); - } -} Index: src/java/org/apache/lucene/index/FormatPostingsFieldsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsFieldsConsumer.java (revision 822088) +++ src/java/org/apache/lucene/index/FormatPostingsFieldsConsumer.java (working copy) @@ -1,36 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -/** Abstract API that consumes terms, doc, freq, prox and - * payloads postings. Concrete implementations of this - * actually do "something" with the postings (write it into - * the index in a specific format). - * - * NOTE: this API is experimental and will likely change - */ -abstract class FormatPostingsFieldsConsumer { - - /** Add a new field */ - abstract FormatPostingsTermsConsumer addField(FieldInfo field) throws IOException; - - /** Called when we are done adding everything. */ - abstract void finish() throws IOException; -} Index: src/java/org/apache/lucene/index/FormatPostingsFieldsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsFieldsWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/FormatPostingsFieldsWriter.java (working copy) @@ -1,73 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.Directory; - -final class FormatPostingsFieldsWriter extends FormatPostingsFieldsConsumer { - - final Directory dir; - final String segment; - final TermInfosWriter termsOut; - final FieldInfos fieldInfos; - final FormatPostingsTermsWriter termsWriter; - final DefaultSkipListWriter skipListWriter; - final int totalNumDocs; - - public FormatPostingsFieldsWriter(SegmentWriteState state, FieldInfos fieldInfos) throws IOException { - super(); - - dir = state.directory; - segment = state.segmentName; - totalNumDocs = state.numDocs; - this.fieldInfos = fieldInfos; - termsOut = new TermInfosWriter(dir, - segment, - fieldInfos, - state.termIndexInterval); - - // TODO: this is a nasty abstraction violation (that we - // peek down to find freqOut/proxOut) -- we need a - // better abstraction here whereby these child consumers - // can provide skip data or not - skipListWriter = new DefaultSkipListWriter(termsOut.skipInterval, - termsOut.maxSkipLevels, - totalNumDocs, - null, - null); - - state.flushedFiles.add(state.segmentFileName(IndexFileNames.TERMS_EXTENSION)); - state.flushedFiles.add(state.segmentFileName(IndexFileNames.TERMS_INDEX_EXTENSION)); - - termsWriter = new FormatPostingsTermsWriter(state, this); - } - - /** Add a new field */ - FormatPostingsTermsConsumer addField(FieldInfo field) { - termsWriter.setField(field); - return termsWriter; - } - - /** Called when we are done adding everything. */ - void finish() throws IOException { - termsOut.close(); - termsWriter.close(); - } -} Index: src/java/org/apache/lucene/index/FormatPostingsPositionsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsPositionsConsumer.java (revision 822088) +++ src/java/org/apache/lucene/index/FormatPostingsPositionsConsumer.java (working copy) @@ -1,32 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.IndexInput; - -abstract class FormatPostingsPositionsConsumer { - - /** Add a new position & payload. If payloadLength > 0 - * you must read those bytes from the IndexInput. */ - abstract void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException; - - /** Called when we are done adding positions & payloads */ - abstract void finish() throws IOException; -} Index: src/java/org/apache/lucene/index/FormatPostingsPositionsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsPositionsWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/FormatPostingsPositionsWriter.java (working copy) @@ -1,87 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; - -import java.io.IOException; - -final class FormatPostingsPositionsWriter extends FormatPostingsPositionsConsumer { - - final FormatPostingsDocsWriter parent; - final IndexOutput out; - - boolean omitTermFreqAndPositions; - boolean storePayloads; - int lastPayloadLength = -1; - - FormatPostingsPositionsWriter(SegmentWriteState state, FormatPostingsDocsWriter parent) throws IOException { - this.parent = parent; - omitTermFreqAndPositions = parent.omitTermFreqAndPositions; - if (parent.parent.parent.fieldInfos.hasProx()) { - // At least one field does not omit TF, so create the - // prox file - final String fileName = IndexFileNames.segmentFileName(parent.parent.parent.segment, IndexFileNames.PROX_EXTENSION); - state.flushedFiles.add(fileName); - out = parent.parent.parent.dir.createOutput(fileName); - parent.skipListWriter.setProxOutput(out); - } else - // Every field omits TF so we will write no prox file - out = null; - } - - int lastPosition; - - /** Add a new position & payload */ - void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException { - assert !omitTermFreqAndPositions: "omitTermFreqAndPositions is true"; - assert out != null; - - final int delta = position - lastPosition; - lastPosition = position; - - if (storePayloads) { - if (payloadLength != lastPayloadLength) { - lastPayloadLength = payloadLength; - out.writeVInt((delta<<1)|1); - out.writeVInt(payloadLength); - } else - out.writeVInt(delta << 1); - if (payloadLength > 0) - out.writeBytes(payload, payloadLength); - } else - out.writeVInt(delta); - } - - void setField(FieldInfo fieldInfo) { - omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions; - storePayloads = omitTermFreqAndPositions ? false : fieldInfo.storePayloads; - } - - /** Called when we are done adding positions & payloads */ - void finish() { - lastPosition = 0; - lastPayloadLength = -1; - } - - void close() throws IOException { - if (out != null) - out.close(); - } -} Index: src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java (revision 822088) +++ src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java (working copy) @@ -1,46 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.ArrayUtil; - -/** - * NOTE: this API is experimental and will likely change - */ - -abstract class FormatPostingsTermsConsumer { - - /** Adds a new term in this field; term ends with U+FFFF - * char */ - abstract FormatPostingsDocsConsumer addTerm(char[] text, int start) throws IOException; - - char[] termBuffer; - FormatPostingsDocsConsumer addTerm(String text) throws IOException { - final int len = text.length(); - if (termBuffer == null || termBuffer.length < 1+len) - termBuffer = new char[ArrayUtil.getNextSize(1+len)]; - text.getChars(0, len, termBuffer, 0); - termBuffer[len] = 0xffff; - return addTerm(termBuffer, 0); - } - - /** Called when we are done adding terms to this field */ - abstract void finish() throws IOException; -} Index: src/java/org/apache/lucene/index/FormatPostingsTermsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsTermsWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/FormatPostingsTermsWriter.java (working copy) @@ -1,71 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -final class FormatPostingsTermsWriter extends FormatPostingsTermsConsumer { - - final FormatPostingsFieldsWriter parent; - final FormatPostingsDocsWriter docsWriter; - final TermInfosWriter termsOut; - FieldInfo fieldInfo; - - FormatPostingsTermsWriter(SegmentWriteState state, FormatPostingsFieldsWriter parent) throws IOException { - super(); - this.parent = parent; - termsOut = parent.termsOut; - docsWriter = new FormatPostingsDocsWriter(state, this); - } - - void setField(FieldInfo fieldInfo) { - this.fieldInfo = fieldInfo; - docsWriter.setField(fieldInfo); - } - - char[] currentTerm; - int currentTermStart; - - long freqStart; - long proxStart; - - /** Adds a new term in this field */ - FormatPostingsDocsConsumer addTerm(char[] text, int start) { - currentTerm = text; - currentTermStart = start; - - // TODO: this is abstraction violation -- ideally this - // terms writer is not so "invasive", looking for file - // pointers in its child consumers. - freqStart = docsWriter.out.getFilePointer(); - if (docsWriter.posWriter.out != null) - proxStart = docsWriter.posWriter.out.getFilePointer(); - - parent.skipListWriter.resetSkip(); - - return docsWriter; - } - - /** Called when we are done adding terms to this field */ - void finish() { - } - - void close() throws IOException { - docsWriter.close(); - } -} Index: src/java/org/apache/lucene/index/FreqProxTermsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/FreqProxTermsWriter.java (working copy) @@ -17,17 +17,19 @@ * limitations under the License. */ -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.UnicodeUtil; - import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Map; -import java.util.ArrayList; -import java.util.List; import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.index.codecs.DocsConsumer; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.PositionsConsumer; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.util.UnicodeUtil; final class FreqProxTermsWriter extends TermsHashConsumer { @@ -60,6 +62,7 @@ void closeDocStore(SegmentWriteState state) {} void abort() {} + private int flushedDocCount; // TODO: would be nice to factor out more of this, eg the // FreqProxFieldMergeState, and code to visit all Fields @@ -71,6 +74,8 @@ // Gather all FieldData's that have postings, across all // ThreadStates List allFields = new ArrayList(); + + flushedDocCount = state.numDocs; Iterator it = threadsAndFields.entrySet().iterator(); while(it.hasNext()) { @@ -88,21 +93,23 @@ } } + final int numAllFields = allFields.size(); + // Sort by field name Collections.sort(allFields); - final int numAllFields = allFields.size(); - // TODO: allow Lucene user to customize this consumer: - final FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos); + // TODO: allow Lucene user to customize this codec: + final FieldsConsumer consumer = state.codec.fieldsConsumer(state); + /* Current writer chain: - FormatPostingsFieldsConsumer - -> IMPL: FormatPostingsFieldsWriter - -> FormatPostingsTermsConsumer - -> IMPL: FormatPostingsTermsWriter - -> FormatPostingsDocConsumer - -> IMPL: FormatPostingsDocWriter - -> FormatPostingsPositionsConsumer + FieldsConsumer + -> IMPL: FormatPostingsTermsDictWriter + -> TermsConsumer + -> IMPL: FormatPostingsTermsDictWriter.TermsWriter + -> DocsConsumer + -> IMPL: FormatPostingsDocsWriter + -> PositionsConsumer -> IMPL: FormatPostingsPositionsWriter */ @@ -145,8 +152,7 @@ FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread) entry.getKey(); perThread.termsHashPerThread.reset(true); } - - consumer.finish(); + consumer.close(); } private byte[] payloadBuffer; @@ -155,7 +161,7 @@ * instances) found in this field and serialize them * into a single RAM segment. */ void appendPostings(FreqProxTermsWriterPerField[] fields, - FormatPostingsFieldsConsumer consumer) + FieldsConsumer consumer) throws CorruptIndexException, IOException { int numFields = fields.length; @@ -172,7 +178,7 @@ assert result; } - final FormatPostingsTermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo); + final TermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo); FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields]; @@ -196,11 +202,18 @@ termStates[numToMerge++] = mergeStates[i]; } - final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(termStates[0].text, termStates[0].textOffset); + final char[] termText = termStates[0].text; + final int termTextOffset = termStates[0].textOffset; + + // nocommit + //System.out.println("FLUSH term=" + new String(termText, termTextOffset, 10)); + + final DocsConsumer docConsumer = termsConsumer.startTerm(termText, termTextOffset); // Now termStates has numToMerge FieldMergeStates // which all share the same term. Now we must // interleave the docID streams. + int numDocs = 0; while(numToMerge > 0) { FreqProxFieldMergeState minState = termStates[0]; @@ -209,8 +222,12 @@ minState = termStates[i]; final int termDocFreq = minState.termFreq; + numDocs++; + + assert minState.docID < flushedDocCount: "doc=" + minState.docID + " maxDoc=" + flushedDocCount; - final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq); + //System.out.println(" docID=" + minState.docID); + final PositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq); final ByteSliceReader prox = minState.prox; @@ -224,6 +241,7 @@ for(int j=0;j> 1; + //System.out.println(" pos=" + position); final int payloadLength; if ((code & 1) != 0) { @@ -241,7 +259,7 @@ posConsumer.addPosition(position, payloadBuffer, 0, payloadLength); } //End for - posConsumer.finish(); + posConsumer.finishDoc(); } if (!minState.nextDoc()) { @@ -269,14 +287,12 @@ } } - docConsumer.finish(); + termsConsumer.finishTerm(termText, termTextOffset, numDocs); } termsConsumer.finish(); } - private final TermInfo termInfo = new TermInfo(); // minimize consing - final UnicodeUtil.UTF8Result termsUTF8 = new UnicodeUtil.UTF8Result(); void files(Collection files) {} Index: src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 822088) +++ src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -17,18 +17,21 @@ * limitations under the License. */ -import org.apache.lucene.store.Directory; - -import java.io.IOException; +import java.io.File; import java.io.FileNotFoundException; +import java.io.FilenameFilter; +import java.io.IOException; import java.io.PrintStream; -import java.util.Map; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Collection; +import java.util.Map; + +import org.apache.lucene.index.codecs.Codecs; +import org.apache.lucene.store.Directory; /* * This class keeps track of each SegmentInfos instance that @@ -121,6 +124,8 @@ infoStream.println("IFD [" + Thread.currentThread().getName() + "]: " + message); } + private final FilenameFilter indexFilenameFilter; + /** * Initialize the deleter: find all previous commits in * the Directory, incref the files they reference, call @@ -129,7 +134,8 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter) + public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter, + Codecs codecs) throws CorruptIndexException, IOException { this.docWriter = docWriter; @@ -144,8 +150,28 @@ // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.getGeneration(); - IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); + final Collection codecsExtensions = codecs.getAllExtensions(); + final FilenameFilter mainFilter = IndexFileNameFilter.getFilter(); + indexFilenameFilter = new FilenameFilter() { + public boolean accept(File dir, String name) { + if (mainFilter.accept(dir, name)) { + return true; + } else { + // See if any of the codecs claim this + // extension: + int i = name.lastIndexOf('.'); + if (i != -1) { + String extension = name.substring(1+i); + if (codecsExtensions.contains(extension)) { + return true; + } + } + return false; + } + } + }; + String[] files = directory.listAll(); CommitPoint currentCommitPoint = null; @@ -154,7 +180,7 @@ String fileName = files[i]; - if (filter.accept(null, fileName) && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { + if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: getRefCount(fileName); @@ -170,7 +196,7 @@ } SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, fileName); + sis.read(directory, fileName, codecs); } catch (FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -207,7 +233,7 @@ // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, segmentInfos.getCurrentSegmentFileName()); + sis.read(directory, segmentInfos.getCurrentSegmentFileName(), codecs); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } @@ -305,7 +331,6 @@ */ public void refresh(String segmentName) throws IOException { String[] files = directory.listAll(); - IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); String segmentPrefix1; String segmentPrefix2; if (segmentName != null) { @@ -318,8 +343,8 @@ for(int i=0;it. * @throws IOException if there is a low-level IO error + * @deprecated Use {@link #docFreq(String,TermRef)} instead. */ public abstract int docFreq(Term t) throws IOException; + /** Returns the number of documents containing the term + * t. This method does not take into + * account deleted documents that have not yet been + * merged away. */ + public int docFreq(String field, TermRef term) throws IOException { + final Terms terms = fields().terms(field); + if (terms != null) { + return terms.docFreq(term); + } else { + return 0; + } + } + /** Returns an enumeration of all the documents which contain * term. For each document, the document number, the frequency of * the term in that document is also provided, for use in @@ -800,6 +895,7 @@ * *

The enumeration is ordered by document number. Each document number * is greater than all that precede it in the enumeration. + * @deprecated Use the new flex API ({@link #termDocsEnum()}) instead. * @throws IOException if there is a low-level IO error */ public TermDocs termDocs(Term term) throws IOException { @@ -809,7 +905,53 @@ return termDocs; } + private static class NullDocsEnum extends DocsEnum { + public int advance(int target) { + return NO_MORE_DOCS; + } + public int next() { + return NO_MORE_DOCS; + } + public int freq() { + return 1; + } + public int read(int[] docs, int[] freqs) { + return 0; + } + public PositionsEnum positions() { + return null; + } + } + private static final NullDocsEnum nullDocsEnum = new NullDocsEnum(); + + // nocommit -- should we return null or NullDocsEnum? + /** Returns DocsEnum for the specified field & term. */ + public DocsEnum termDocsEnum(Bits skipDocs, String field, TermRef term) throws IOException { + + assert field != null; + assert term != null; + + final Terms terms = fields().terms(field); + if (terms != null) { + if (Codec.DEBUG) { + System.out.println("ir.termDocsEnum field=" + field + " terms=" + terms + " this=" + this); + } + final DocsEnum docs = terms.docs(skipDocs, term); + if (Codec.DEBUG) { + System.out.println("ir.termDocsEnum field=" + field + " docs=" +docs); + } + if (docs != null) { + return docs; + } else { + return nullDocsEnum; + } + } else { + return nullDocsEnum; + } + } + /** Returns an unpositioned {@link TermDocs} enumerator. + * @deprecated Use the new flex API ({@link #fields()}) instead. * @throws IOException if there is a low-level IO error */ public abstract TermDocs termDocs() throws IOException; @@ -829,6 +971,8 @@ *

This positional information facilitates phrase and proximity searching. *

The enumeration is ordered by document number. Each document number is * greater than all that precede it in the enumeration. + * @deprecated Please switch the flex API ({@link + * #termDocsEnum()}) instead * @throws IOException if there is a low-level IO error */ public TermPositions termPositions(Term term) throws IOException { @@ -839,6 +983,8 @@ } /** Returns an unpositioned {@link TermPositions} enumerator. + * @deprecated Please switch the flex API ({@link + * #termDocsEnum()}) instead * @throws IOException if there is a low-level IO error */ public abstract TermPositions termPositions() throws IOException; @@ -846,7 +992,7 @@ /** Deletes the document numbered docNum. Once a document is - * deleted it will not appear in TermDocs or TermPostitions enumerations. + * deleted it will not appear in TermDocs or TermPositions enumerations. * Attempts to read its field with the {@link #document} * method will result in an error. The presence of this document may still be * reflected in the {@link #docFreq} statistic, though @@ -1022,6 +1168,31 @@ */ public abstract Collection getFieldNames(FieldOption fldOption); + private final class DeletedDocsBits implements Bits { + public boolean get(int docID) { + return isDeleted(docID); + } + } + + public Bits getDeletedDocs() throws IOException { + return new DeletedDocsBits(); + } + + + /** + * Forcibly unlocks the index in the named directory. + *

+ * Caution: this should only be used by failure recovery code, + * when it is known that no other process nor thread is in fact + * currently accessing this index. + * @deprecated Please use {@link IndexWriter#unlock(Directory)} instead. + * This method will be removed in the 3.0 release. + * + */ + public static void unlock(Directory directory) throws IOException { + directory.makeLock(IndexWriter.WRITE_LOCK_NAME).release(); + } + /** * Expert: return the IndexCommit that this reader has * opened. This method is only implemented by those @@ -1167,7 +1338,16 @@ * #getSequentialSubReaders} and ask each sub reader for * its unique term count. */ public long getUniqueTermCount() throws IOException { - throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()"); + long numTerms = 0; + FieldsEnum it = fields().iterator(); + while(true) { + String field = it.next(); + if (field == null) { + break; + } + numTerms += fields().terms(field).getUniqueTermCount(); + } + return numTerms; } /** Expert: Return the state of the flag that disables fakes norms in favor of representing the absence of field norms with null. Index: src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/IndexWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -23,14 +23,13 @@ import org.apache.lucene.search.Similarity; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.util.Constants; +import org.apache.lucene.index.codecs.Codecs; -import java.io.File; import java.io.IOException; import java.io.PrintStream; import java.util.List; @@ -346,6 +345,7 @@ private int termIndexInterval = DEFAULT_TERM_INDEX_INTERVAL; + private boolean closeDir; private boolean closed; private boolean closing; @@ -466,7 +466,7 @@ // reader; in theory we could do similar retry logic, // just like we do when loading segments_N synchronized(this) { - return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor); + return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor, codecs); } } @@ -682,14 +682,14 @@ if (doOpenStores) { sr.openDocStores(); } - if (termsIndexDivisor != -1 && !sr.termsIndexLoaded()) { + if (termsIndexDivisor != -1) { // If this reader was originally opened because we // needed to merge it, we didn't load the terms // index. But now, if the caller wants the terms // index (eg because it's doing deletes, or an NRT // reader is being opened) we ask the reader to // load its terms index. - sr.loadTermsIndex(termsIndexDivisor); + sr.loadTermsIndex(); } } @@ -939,7 +939,7 @@ */ public IndexWriter(Directory d, Analyzer a, boolean create, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, null, false, mfl.getLimit(), null, null); + init(d, a, create, false, null, false, mfl.getLimit(), null, null, null); } /** @@ -967,9 +967,10 @@ */ public IndexWriter(Directory d, Analyzer a, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, create, false, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } + /** * Constructs an IndexWriter for the index in * d, first creating it if it does not @@ -994,7 +995,7 @@ */ public IndexWriter(Directory d, Analyzer a, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, null, false, mfl.getLimit(), null, null); + init(d, a, false, null, false, mfl.getLimit(), null, null); } /** @@ -1019,7 +1020,7 @@ */ public IndexWriter(Directory d, Analyzer a) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, false, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null); } /** @@ -1045,7 +1046,7 @@ */ public IndexWriter(Directory d, boolean autoCommit, Analyzer a) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, null, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, false, null, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); } /** @@ -1075,7 +1076,7 @@ */ public IndexWriter(Directory d, boolean autoCommit, Analyzer a, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, null, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, create, false, null, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } /** @@ -1102,7 +1103,7 @@ */ public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, deletionPolicy, false, mfl.getLimit(), null, null); + init(d, a, false, deletionPolicy, false, mfl.getLimit(), null, null); } /** @@ -1129,7 +1130,7 @@ */ public IndexWriter(Directory d, boolean autoCommit, Analyzer a, IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, deletionPolicy, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, false, deletionPolicy, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); } /** @@ -1162,7 +1163,7 @@ */ public IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, deletionPolicy, false, mfl.getLimit(), null, null); + init(d, a, create, false, deletionPolicy, false, mfl.getLimit(), null, null, null); } /** @@ -1197,9 +1198,9 @@ * false or if there is any other low-level * IO error */ - IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexingChain indexingChain, IndexCommit commit) + IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexingChain indexingChain, IndexCommit commit, Codecs codecs) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, deletionPolicy, false, mfl.getLimit(), indexingChain, commit); + init(d, a, create, false, deletionPolicy, false, mfl.getLimit(), indexingChain, commit, codecs); } /** @@ -1232,7 +1233,7 @@ */ public IndexWriter(Directory d, boolean autoCommit, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, deletionPolicy, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, create, false, deletionPolicy, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } /** @@ -1273,23 +1274,31 @@ */ public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexCommit commit) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, false, deletionPolicy, false, mfl.getLimit(), null, commit); + init(d, a, false, false, deletionPolicy, false, mfl.getLimit(), null, commit, null); } + + Codecs codecs; - private void init(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, + private void init(Directory d, Analyzer a, boolean closeDir, IndexDeletionPolicy deletionPolicy, boolean autoCommit, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit) throws CorruptIndexException, LockObtainFailedException, IOException { if (IndexReader.indexExists(d)) { - init(d, a, false, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit); + init(d, a, false, closeDir, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit, null); } else { - init(d, a, true, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit); + init(d, a, true, closeDir, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit, null); } } - private void init(Directory d, Analyzer a, final boolean create, + private void init(Directory d, Analyzer a, final boolean create, boolean closeDir, IndexDeletionPolicy deletionPolicy, boolean autoCommit, int maxFieldLength, - IndexingChain indexingChain, IndexCommit commit) + IndexingChain indexingChain, IndexCommit commit, Codecs codecsIn) throws CorruptIndexException, LockObtainFailedException, IOException { + this.closeDir = closeDir; + if (codecs == null) { + codecs = Codecs.getDefault(); + } else { + codecs = codecsIn; + } directory = d; analyzer = a; setMessageID(defaultInfoStream); @@ -1316,7 +1325,7 @@ // segments_N file with no segments: boolean doCommit; try { - segmentInfos.read(directory); + segmentInfos.read(directory, codecs); segmentInfos.clear(); doCommit = false; } catch (IOException e) { @@ -1336,7 +1345,7 @@ changeCount++; } } else { - segmentInfos.read(directory); + segmentInfos.read(directory, codecs); if (commit != null) { // Swap out all segments, but, keep metadata in @@ -1347,7 +1356,7 @@ if (commit.getDirectory() != directory) throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory"); SegmentInfos oldInfos = new SegmentInfos(); - oldInfos.read(directory, commit.getSegmentsFileName()); + oldInfos.read(directory, commit.getSegmentsFileName(), codecs); segmentInfos.replace(oldInfos); changeCount++; if (infoStream != null) @@ -1370,7 +1379,7 @@ // KeepOnlyLastCommitDeleter: deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, infoStream, docWriter); + segmentInfos, infoStream, docWriter, this.codecs); if (deleter.startingCommitDeleted) // Deletion policy deleted the "head" commit point. @@ -1969,6 +1978,9 @@ deleter.close(); } + if (closeDir) + directory.close(); + if (writeLock != null) { writeLock.release(); // release write lock writeLock = null; @@ -3306,7 +3318,7 @@ ensureOpen(); for (int i = 0; i < dirs.length; i++) { SegmentInfos sis = new SegmentInfos(); // read infos from dir - sis.read(dirs[i]); + sis.read(dirs[i], codecs); for (int j = 0; j < sis.size(); j++) { final SegmentInfo info = sis.info(j); docCount += info.docCount; @@ -3436,7 +3448,7 @@ } SegmentInfos sis = new SegmentInfos(); // read infos from dir - sis.read(dirs[i]); + sis.read(dirs[i], codecs); for (int j = 0; j < sis.size(); j++) { SegmentInfo info = sis.info(j); assert !segmentInfos.contains(info): "dup info dir=" + info.dir + " name=" + info.name; @@ -3619,10 +3631,11 @@ // call hits an exception it will release the write // lock: startTransaction(true); - + success = false; + try { mergedName = newSegmentName(); - merger = new SegmentMerger(this, mergedName, null); + merger = new SegmentMerger(this, mergedName, null, codecs); SegmentReader sReader = null; synchronized(this) { @@ -3645,7 +3658,7 @@ synchronized(this) { segmentInfos.clear(); // pop old infos & add new info = new SegmentInfo(mergedName, docCount, directory, false, true, - -1, null, false, merger.hasProx()); + -1, null, false, merger.hasProx(), merger.getCodec()); setDiagnostics(info, "addIndexes(IndexReader[])"); segmentInfos.add(info); } @@ -3692,7 +3705,7 @@ startTransaction(false); try { - merger.createCompoundFile(mergedName + ".cfs"); + merger.createCompoundFile(mergedName + ".cfs", info); synchronized(this) { info.setUseCompoundFile(true); } @@ -4068,7 +4081,9 @@ directory, false, true, docStoreOffset, docStoreSegment, docStoreIsCompoundFile, - docWriter.hasProx()); + docWriter.hasProx(), + docWriter.getCodec()); + setDiagnostics(newSegment, "flush"); } @@ -4284,7 +4299,8 @@ } } - merge.info.setHasProx(merger.hasProx()); + // mxx + // System.out.println(Thread.currentThread().getName() + ": finish setHasProx=" + merger.hasProx() + " seg=" + merge.info.name); segmentInfos.subList(start, start + merge.segments.size()).clear(); assert !segmentInfos.contains(merge.info); @@ -4585,7 +4601,8 @@ docStoreOffset, docStoreSegment, docStoreIsCompoundFile, - false); + false, + null); Map details = new HashMap(); @@ -4703,7 +4720,7 @@ if (infoStream != null) message("merging " + merge.segString(directory)); - merger = new SegmentMerger(this, mergedName, merge); + merger = new SegmentMerger(this, mergedName, merge, codecs); merge.readers = new SegmentReader[numSegments]; merge.readersClone = new SegmentReader[numSegments]; @@ -4776,8 +4793,17 @@ // This is where all the work happens: mergedDocCount = merge.info.docCount = merger.merge(merge.mergeDocStores); + // Record which codec was used to write the segment + merge.info.setCodec(merger.getCodec()); + assert mergedDocCount == totDocCount; + // Very important to do this before opening the reader + // because codec must know if prox was written for + // this segment: + //System.out.println("merger set hasProx=" + merger.hasProx() + " seg=" + merge.info.name); + merge.info.setHasProx(merger.hasProx()); + // TODO: in the non-realtime case, we may want to only // keep deletes (it's costly to open entire reader // when we just need deletes) @@ -4816,7 +4842,7 @@ } catch (Throwable t) { } // This was a private clone and we had the only reference - assert merge.readersClone[i].getRefCount() == 0; + // assert merge.readersClone[i].getRefCount() == 0: "refCount should be 0 but is " + merge.readersClone[i].getRefCount(); } } } else { @@ -4828,7 +4854,7 @@ if (merge.readersClone[i] != null) { merge.readersClone[i].close(); // This was a private clone and we had the only reference - assert merge.readersClone[i].getRefCount() == 0; + //assert merge.readersClone[i].getRefCount() == 0; } } } @@ -4859,7 +4885,7 @@ final String compoundFileName = mergedName + "." + IndexFileNames.COMPOUND_FILE_EXTENSION; try { - merger.createCompoundFile(compoundFileName); + merger.createCompoundFile(compoundFileName, merge.info); success = true; } catch (IOException ioe) { synchronized(this) { Index: src/java/org/apache/lucene/index/LegacyFields.java =================================================================== --- src/java/org/apache/lucene/index/LegacyFields.java (revision 0) +++ src/java/org/apache/lucene/index/LegacyFields.java (revision 0) @@ -0,0 +1,45 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** Implements new API (FieldsEnum/TermsEnum) on top of old + * API. Used only for IndexReader impls outside Lucene's + * core. */ +class LegacyFields extends Fields { + private final IndexReader r; + private TermEnum terms; + + public LegacyFields(IndexReader r) throws IOException { + this.r = r; + } + + public FieldsEnum iterator() throws IOException { + return new LegacyFieldsEnum(r); + } + + public Terms terms(String field) throws IOException { + // nocommit + return new LegacyTerms(r, field); + } + + public void close() throws IOException { + // nocommit + } +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/index/LegacyFields.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/LegacyFieldsEnum.java =================================================================== --- src/java/org/apache/lucene/index/LegacyFieldsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/LegacyFieldsEnum.java (revision 0) @@ -0,0 +1,214 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.util.Bits; + +/** Implements new API (FieldsEnum/TermsEnum) on top of old + * API. Used only for IndexReader impls outside Lucene's + * core. */ +class LegacyFieldsEnum extends FieldsEnum { + private final IndexReader r; + private TermEnum terms; + private String field; + + public LegacyFieldsEnum(IndexReader r) throws IOException { + this.r = r; + terms = r.terms(); + } + + private void doSeek(Term t) throws IOException { + terms.close(); + terms = r.terms(t); + } + + /* + public boolean seek(String field) throws IOException { + this.field = field; + doSeek(new Term(field, "")); + return terms.term() != null && terms.term().field.equals(field); + } + */ + + public String next() throws IOException { + + final Term seekTo = new Term(field, "\uFFFF"); + + doSeek(seekTo); + if (terms.term() != null) { + String newField = terms.term().field; + assert !newField.equals(field); + field = newField; + return field; + } else { + return null; + } + } + + public TermsEnum terms() throws IOException { + return new LegacyTermsEnum(r, field); + } + + public void close() throws IOException { + terms.close(); + } + + // Emulates flex on top of legacy API + static class LegacyTermsEnum extends TermsEnum { + private final IndexReader r; + private final String field; + private TermEnum terms; + private final SeekResult seekResult = new SeekResult(); + + LegacyTermsEnum(IndexReader r, String field) throws IOException { + this.r = r; + this.field = field; + this.terms = r.terms(new Term(field, "")); + } + + public SeekResult seek(TermRef text) throws IOException { + terms.close(); + terms = r.terms(new Term(field, text.toString())); + final Term t = terms.term(); + if (t == null) { + seekResult.status = SeekResult.Status.END; + } else if (text.termEquals(new TermRef(t.text()))) { + seekResult.status = SeekResult.Status.FOUND; + } else { + seekResult.status = SeekResult.Status.NOT_FOUND; + // nocommit reuse TermRef instance + seekResult.term = new TermRef(t.text()); + } + return seekResult; + } + + public TermRef next() throws IOException { + if (terms.next()) { + // nocommit -- reuse TermRef instance + return new TermRef(terms.term().text()); + } else { + return null; + } + } + + public String text() { + return terms.term().text; + } + + public int docFreq() { + return terms.docFreq(); + } + + public DocsEnum docs(Bits skipDocs) throws IOException { + return new LegacyDocsEnum(r, field, terms.term(), skipDocs); + } + + public void close() throws IOException { + terms.close(); + } + } + + // Emulates flex on top of legacy API + private static class LegacyDocsEnum extends DocsEnum { + final TermDocs td; + final Term term; + final IndexReader r; + final String field; + final Bits skipDocs; + + TermPositions tp; + + LegacyDocsEnum(IndexReader r, String field, Term term, Bits skipDocs) throws IOException { + this.r = r; + this.field = field; + this.term = term; + td = r.termDocs(term); + this.skipDocs = skipDocs; + } + + // nocommit -- must enforce skipDocs... but old API will + // always secretly skip deleted docs, and we can't work + // around that for external readers? + public int next() throws IOException { + if (td.next()) { + return td.doc(); + } else { + return NO_MORE_DOCS; + } + } + + public int advance(int target) throws IOException { + if (td.skipTo(target)) { + return td.doc(); + } else { + return NO_MORE_DOCS; + } + } + + public int freq() { + return td.freq(); + } + + public int read(int[] docs, int[] freqs) throws IOException { + return td.read(docs, freqs); + } + + public void close() throws IOException { + td.close(); + } + + LegacyPositionsEnum lpe; + + public PositionsEnum positions() throws IOException { + if (tp == null) { + tp = r.termPositions(term); + lpe = new LegacyPositionsEnum(tp); + } else { + tp.seek(term); + } + return lpe; + } + } + + // Emulates flex on top of legacy API + private static class LegacyPositionsEnum extends PositionsEnum { + + final TermPositions tp; + + LegacyPositionsEnum(TermPositions tp) { + this.tp = tp; + } + + public int next() throws IOException { + return tp.nextPosition(); + } + + public int getPayloadLength() { + return tp.getPayloadLength(); + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + return tp.getPayload(data, offset); + } + + public boolean hasPayload() { + return tp.isPayloadAvailable(); + } + } +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/index/LegacyFieldsEnum.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java =================================================================== --- src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java (revision 0) +++ src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java (revision 0) @@ -0,0 +1,85 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +final class LegacySegmentMergeInfo { + Term term; + int base; + int ord; // the position of the segment in a MultiReader + TermEnum termEnum; + IndexReader reader; + int delCount; + private TermPositions postings; // use getPositions() + private int[] docMap; // use getDocMap() + + LegacySegmentMergeInfo(int b, TermEnum te, IndexReader r) + throws IOException { + base = b; + reader = r; + termEnum = te; + term = te.term(); + } + + // maps around deleted docs + int[] getDocMap() { + if (docMap == null) { + delCount = 0; + // build array which maps document numbers around deletions + if (reader.hasDeletions()) { + int maxDoc = reader.maxDoc(); + docMap = new int[maxDoc]; + int j = 0; + for (int i = 0; i < maxDoc; i++) { + if (reader.isDeleted(i)) { + delCount++; + docMap[i] = -1; + } else + docMap[i] = j++; + } + } + } + return docMap; + } + + TermPositions getPositions() throws IOException { + if (postings == null) { + postings = reader.termPositions(); + } + return postings; + } + + final boolean next() throws IOException { + if (termEnum.next()) { + term = termEnum.term(); + return true; + } else { + term = null; + return false; + } + } + + final void close() throws IOException { + termEnum.close(); + if (postings != null) { + postings.close(); + } +} +} + Property changes on: src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/LegacySegmentMergeQueue.java =================================================================== --- src/java/org/apache/lucene/index/LegacySegmentMergeQueue.java (revision 0) +++ src/java/org/apache/lucene/index/LegacySegmentMergeQueue.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.util.PriorityQueue; + +final class LegacySegmentMergeQueue extends PriorityQueue { + LegacySegmentMergeQueue(int size) { + initialize(size); + } + + protected final boolean lessThan(LegacySegmentMergeInfo a, LegacySegmentMergeInfo b) { + int comparison = a.term.compareTo(b.term); + if (comparison == 0) + return a.base < b.base; + else + return comparison < 0; + } + + final void close() throws IOException { + while (top() != null) + ((LegacySegmentMergeInfo)pop()).close(); + } + +} Property changes on: src/java/org/apache/lucene/index/LegacySegmentMergeQueue.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/LegacyTerms.java =================================================================== --- src/java/org/apache/lucene/index/LegacyTerms.java (revision 0) +++ src/java/org/apache/lucene/index/LegacyTerms.java (revision 0) @@ -0,0 +1,45 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.IOException; + +/** Implements new API (FieldsEnum/TermsEnum) on top of old + * API. Used only for IndexReader impls outside Lucene's + * core. */ +class LegacyTerms extends Terms { + + private final IndexReader r; + private final String field; + + LegacyTerms(IndexReader r, String field) { + this.r = r; + this.field = field; + } + + public TermsEnum iterator() throws IOException { + return new LegacyFieldsEnum.LegacyTermsEnum(r, field); + } + + public void close() { + } +} + + + Property changes on: src/java/org/apache/lucene/index/LegacyTerms.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/MultiReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiReader.java (revision 822088) +++ src/java/org/apache/lucene/index/MultiReader.java (working copy) @@ -25,10 +25,13 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.DirectoryReader.MultiBits; +import org.apache.lucene.index.DirectoryReader.MultiFields; import org.apache.lucene.index.DirectoryReader.MultiTermDocs; import org.apache.lucene.index.DirectoryReader.MultiTermEnum; import org.apache.lucene.index.DirectoryReader.MultiTermPositions; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.util.Bits; /** An IndexReader which reads multiple indexes, appending their content. * @@ -42,6 +45,8 @@ private int maxDoc = 0; private int numDocs = -1; private boolean hasDeletions = false; + private MultiBits deletedDocs; + private MultiFields fields; /** *

Construct a MultiReader aggregating the named set of (sub)readers. @@ -51,7 +56,7 @@ * @param subReaders set of (sub)readers * @throws IOException */ - public MultiReader(IndexReader[] subReaders) { + public MultiReader(IndexReader[] subReaders) throws IOException { initialize(subReaders, true); } @@ -64,14 +69,15 @@ * @param subReaders set of (sub)readers * @throws IOException */ - public MultiReader(IndexReader[] subReaders, boolean closeSubReaders) { + public MultiReader(IndexReader[] subReaders, boolean closeSubReaders) throws IOException { initialize(subReaders, closeSubReaders); } - private void initialize(IndexReader[] subReaders, boolean closeSubReaders) { + private void initialize(IndexReader[] subReaders, boolean closeSubReaders) throws IOException { this.subReaders = (IndexReader[]) subReaders.clone(); starts = new int[subReaders.length + 1]; // build starts array decrefOnClose = new boolean[subReaders.length]; + Bits[] subs = new Bits[subReaders.length]; for (int i = 0; i < subReaders.length; i++) { starts[i] = maxDoc; maxDoc += subReaders[i].maxDoc(); // compute maxDocs @@ -83,12 +89,24 @@ decrefOnClose[i] = false; } - if (subReaders[i].hasDeletions()) + if (subReaders[i].hasDeletions()) { hasDeletions = true; + } + subs[i] = subReaders[i].getDeletedDocs(); } starts[subReaders.length] = maxDoc; + if (hasDeletions) { + deletedDocs = new MultiBits(subs, starts); + } else { + deletedDocs = null; + } + fields = new MultiFields(subReaders, starts); } - + + public Fields fields() throws IOException { + return fields; + } + /** * Tries to reopen the subreaders. *
@@ -129,6 +147,10 @@ } } + public Bits getDeletedDocs() { + return deletedDocs; + } + /** * If clone is true then we clone each of the subreaders * @param doClone @@ -345,6 +367,15 @@ return total; } + public int docFreq(String field, TermRef t) throws IOException { + ensureOpen(); + int total = 0; // sum freqs in segments + for (int i = 0; i < subReaders.length; i++) { + total += subReaders[i].docFreq(field, t); + } + return total; + } + public TermDocs termDocs() throws IOException { ensureOpen(); return new MultiTermDocs(this, subReaders, starts); Index: src/java/org/apache/lucene/index/MultipleTermPositions.java =================================================================== --- src/java/org/apache/lucene/index/MultipleTermPositions.java (revision 822088) +++ src/java/org/apache/lucene/index/MultipleTermPositions.java (working copy) @@ -28,7 +28,8 @@ /** * Allows you to iterate over the {@link TermPositions} for multiple {@link Term}s as * a single {@link TermPositions}. - * + * @deprecated This class is being replaced by the package + * private MultiDocsEnum on org.apache.lucene.search. */ public class MultipleTermPositions implements TermPositions { Index: src/java/org/apache/lucene/index/ParallelReader.java =================================================================== --- src/java/org/apache/lucene/index/ParallelReader.java (revision 822088) +++ src/java/org/apache/lucene/index/ParallelReader.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.util.Bits; import java.io.IOException; import java.util.*; @@ -47,7 +48,7 @@ private List readers = new ArrayList(); private List decrefOnClose = new ArrayList(); // remember which subreaders to decRef on close boolean incRefReaders = false; - private SortedMap fieldToReader = new TreeMap(); + private SortedMap< String, IndexReader> fieldToReader = new TreeMap(); private Map readerToFields = new HashMap(); private List storedFieldReaders = new ArrayList(); @@ -55,6 +56,8 @@ private int numDocs; private boolean hasDeletions; + private ParallelFields fields = new ParallelFields(); + /** Construct a ParallelReader. *

Note that all subreaders are closed if this ParallelReader is closed.

*/ @@ -109,8 +112,10 @@ Iterator i = fields.iterator(); while (i.hasNext()) { // update fieldToReader map String field = (String)i.next(); - if (fieldToReader.get(field) == null) + if (fieldToReader.get(field) == null) { fieldToReader.put(field, reader); + } + this.fields.addField(field, reader); } if (!ignoreStoredFields) @@ -122,6 +127,57 @@ } decrefOnClose.add(Boolean.valueOf(incRefReaders)); } + + private class ParallelFieldsEnum extends FieldsEnum { + String currentField; + IndexReader currentReader; + Iterator keys; + private final HashMap readerFields = new HashMap(); + + ParallelFieldsEnum() { + keys = fieldToReader.keySet().iterator(); + } + + public String next() throws IOException { + if (keys.hasNext()) { + currentField = (String) keys.next(); + currentReader = (IndexReader) fieldToReader.get(currentField); + } else { + currentField = null; + currentReader = null; + } + return currentField; + } + + public TermsEnum terms() throws IOException { + assert currentReader != null; + return currentReader.fields().terms(currentField).iterator(); + } + } + + // Single instance of this, per ParallelReader instance + private class ParallelFields extends Fields { + final HashMap fields = new HashMap(); + + public void addField(String field, IndexReader r) throws IOException { + fields.put(field, r.fields().terms(field)); + } + + public FieldsEnum iterator() throws IOException { + return new ParallelFieldsEnum(); + } + public Terms terms(String field) throws IOException { + return fields.get(field); + } + } + + public Bits getDeletedDocs() throws IOException { + return ((IndexReader) readers.get(0)).getDeletedDocs(); + } + + public Fields fields() { + return fields; + } public synchronized Object clone() { try { @@ -374,6 +430,12 @@ return reader==null ? 0 : reader.docFreq(term); } + public int docFreq(String field, TermRef term) throws IOException { + ensureOpen(); + IndexReader reader = ((IndexReader)fieldToReader.get(field)); + return reader == null? 0 : reader.docFreq(field, term); + } + public TermDocs termDocs(Term term) throws IOException { ensureOpen(); return new ParallelTermDocs(term); @@ -468,7 +530,7 @@ private class ParallelTermEnum extends TermEnum { private String field; - private Iterator fieldIterator; + private Iterator fieldIterator; private TermEnum termEnum; public ParallelTermEnum() throws IOException { @@ -479,12 +541,12 @@ return; } if (field != null) - termEnum = ((IndexReader)fieldToReader.get(field)).terms(); + termEnum = fieldToReader.get(field).terms(); } public ParallelTermEnum(Term term) throws IOException { field = term.field(); - IndexReader reader = ((IndexReader)fieldToReader.get(field)); + IndexReader reader = fieldToReader.get(field); if (reader!=null) termEnum = reader.terms(term); } @@ -506,7 +568,7 @@ } while (fieldIterator.hasNext()) { field = (String) fieldIterator.next(); - termEnum = ((IndexReader)fieldToReader.get(field)).terms(new Term(field)); + termEnum = fieldToReader.get(field).terms(new Term(field)); Term term = termEnum.term(); if (term!=null && term.field()==field) return true; Index: src/java/org/apache/lucene/index/PositionsEnum.java =================================================================== --- src/java/org/apache/lucene/index/PositionsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/PositionsEnum.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.AttributeSource; + +public abstract class PositionsEnum extends AttributeSource { + + // nocommit + public String desc; + + /** Returns the next position. You should only call this + * up to {@link FormatPostingsDocsEnum#freq()} times else + * the behavior is not defined. */ + public abstract int next() throws IOException; + + public abstract int getPayloadLength(); + + // nocommit -- improve this so that readers that do their + // own buffering can save a copy + public abstract byte[] getPayload(byte[] data, int offset) throws IOException; + + public abstract boolean hasPayload(); +} Property changes on: src/java/org/apache/lucene/index/PositionsEnum.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java =================================================================== --- src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java (revision 822088) +++ src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java (working copy) @@ -18,22 +18,23 @@ */ import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.Codecs; import java.io.IOException; import java.util.Map; class ReadOnlyDirectoryReader extends DirectoryReader { - ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor) throws IOException { - super(directory, sis, deletionPolicy, true, termInfosIndexDivisor); + ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor, Codecs codecs) throws IOException { + super(directory, sis, deletionPolicy, true, termInfosIndexDivisor, codecs); } ReadOnlyDirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache, boolean doClone, - int termInfosIndexDivisor) throws IOException { - super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor); + int termInfosIndexDivisor, Codecs codecs) throws IOException { + super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor, codecs); } - ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException { - super(writer, infos, termInfosIndexDivisor); + ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException { + super(writer, infos, termInfosIndexDivisor, codecs); } protected void acquireWriteLock() { Index: src/java/org/apache/lucene/index/SegmentFieldMergeQueue.java =================================================================== --- src/java/org/apache/lucene/index/SegmentFieldMergeQueue.java (revision 0) +++ src/java/org/apache/lucene/index/SegmentFieldMergeQueue.java (revision 0) @@ -0,0 +1,34 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.PriorityQueue; + +// Used to merge-sort by SegmentMergeInfo.field +final class SegmentFieldMergeQueue extends PriorityQueue { + SegmentFieldMergeQueue(int size) { + initialize(size); + } + + protected final boolean lessThan(Object a, Object b) { + SegmentMergeInfo stiA = (SegmentMergeInfo)a; + SegmentMergeInfo stiB = (SegmentMergeInfo)b; + // nocommit ok not to break ties? + return stiA.field.compareTo(stiB.field) < 0; + } +} Property changes on: src/java/org/apache/lucene/index/SegmentFieldMergeQueue.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- src/java/org/apache/lucene/index/SegmentInfo.java (revision 822088) +++ src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -21,6 +21,8 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BitVector; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.Codecs; import java.io.IOException; import java.util.List; import java.util.Map; @@ -88,6 +90,11 @@ // (if it's an older index) private boolean hasProx; // True if this segment has any fields with omitTermFreqAndPositions==false + + // nocommit: unread field + private boolean flexPostings; // True if postings were written with new flex format + private Codec codec; + private Map diagnostics; @@ -95,7 +102,7 @@ return "si: "+dir.toString()+" "+name+" docCount: "+docCount+" delCount: "+delCount+" delFileName: "+getDelFileName(); } - public SegmentInfo(String name, int docCount, Directory dir) { + public SegmentInfo(String name, int docCount, Directory dir, Codec codec) { this.name = name; this.docCount = docCount; this.dir = dir; @@ -108,15 +115,21 @@ docStoreIsCompoundFile = false; delCount = 0; hasProx = true; + flexPostings = true; + this.codec = codec; } + // nocommit -- this ctor is only used by back-compat tests public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile) { - this(name, docCount, dir, isCompoundFile, hasSingleNormFile, -1, null, false, true); + this(name, docCount, dir, isCompoundFile, hasSingleNormFile, -1, null, false, true, null); + SegmentWriteState state = new SegmentWriteState(null, dir, name, null, null, docCount, docCount, -1, Codecs.getDefault()); + codec = state.codec = Codecs.getDefault().getWriter(state); } - - public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile, - int docStoreOffset, String docStoreSegment, boolean docStoreIsCompoundFile, boolean hasProx) { - this(name, docCount, dir); + + public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile, + int docStoreOffset, String docStoreSegment, boolean docStoreIsCompoundFile, boolean hasProx, + Codec codec) { + this(name, docCount, dir, codec); this.isCompoundFile = (byte) (isCompoundFile ? YES : NO); this.hasSingleNormFile = hasSingleNormFile; preLockless = false; @@ -124,6 +137,7 @@ this.docStoreSegment = docStoreSegment; this.docStoreIsCompoundFile = docStoreIsCompoundFile; this.hasProx = hasProx; + this.codec = codec; delCount = 0; assert docStoreOffset == -1 || docStoreSegment != null: "dso=" + docStoreOffset + " dss=" + docStoreSegment + " docCount=" + docCount; } @@ -149,6 +163,7 @@ isCompoundFile = src.isCompoundFile; hasSingleNormFile = src.hasSingleNormFile; delCount = src.delCount; + codec = src.codec; } // must be Map @@ -169,10 +184,11 @@ * @param format format of the segments info file * @param input input handle to read segment info from */ - SegmentInfo(Directory dir, int format, IndexInput input) throws IOException { + SegmentInfo(Directory dir, int format, IndexInput input, Codecs codecs) throws IOException { this.dir = dir; name = input.readString(); docCount = input.readInt(); + final String codecName; if (format <= SegmentInfos.FORMAT_LOCKLESS) { delGen = input.readLong(); if (format <= SegmentInfos.FORMAT_SHARED_DOC_STORE) { @@ -215,6 +231,13 @@ else hasProx = true; + // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name); + + if (format <= SegmentInfos.FORMAT_FLEX_POSTINGS) + codecName = input.readString(); + else + codecName = "PreFlex"; + if (format <= SegmentInfos.FORMAT_DIAGNOSTICS) { diagnostics = input.readStringStringMap(); } else { @@ -231,8 +254,10 @@ docStoreSegment = null; delCount = -1; hasProx = true; + codecName = "PreFlex"; diagnostics = Collections.EMPTY_MAP; } + codec = codecs.lookup(codecName); } void setNumFields(int numFields) { @@ -315,7 +340,7 @@ } public Object clone () { - SegmentInfo si = new SegmentInfo(name, docCount, dir); + SegmentInfo si = new SegmentInfo(name, docCount, dir, codec); si.isCompoundFile = isCompoundFile; si.delGen = delGen; si.delCount = delCount; @@ -329,6 +354,7 @@ si.docStoreOffset = docStoreOffset; si.docStoreSegment = docStoreSegment; si.docStoreIsCompoundFile = docStoreIsCompoundFile; + si.codec = codec; return si; } @@ -560,6 +586,9 @@ output.writeByte(isCompoundFile); output.writeInt(delCount); output.writeByte((byte) (hasProx ? 1:0)); + // mxx + //System.out.println(Thread.currentThread().getName() + ": si.write hasProx=" + hasProx + " seg=" + name); + output.writeString(codec.name); output.writeStringStringMap(diagnostics); } @@ -572,6 +601,19 @@ return hasProx; } + /** Can only be called once. */ + public void setCodec(Codec codec) { + assert this.codec == null; + if (codec == null) { + throw new IllegalArgumentException("codec must be non-null"); + } + this.codec = codec; + } + + Codec getCodec() { + return codec; + } + private void addIfExists(List files, String fileName) throws IOException { if (dir.fileExists(fileName)) files.add(fileName); @@ -598,8 +640,12 @@ files.add(name + "." + IndexFileNames.COMPOUND_FILE_EXTENSION); } else { final String[] exts = IndexFileNames.NON_STORE_INDEX_EXTENSIONS; - for(int i=0;i 0; i--) { // read segmentInfos - add(new SegmentInfo(directory, format, input)); + add(new SegmentInfo(directory, format, input, codecs)); } if(format >= 0){ // in old format the version number may be at the end of the file @@ -300,13 +306,16 @@ * @throws IOException if there is a low-level IO error */ public final void read(Directory directory) throws CorruptIndexException, IOException { - + read(directory, Codecs.getDefault()); + } + + public final void read(Directory directory, final Codecs codecs) throws CorruptIndexException, IOException { generation = lastGeneration = -1; new FindSegmentsFile(directory) { protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - read(directory, segmentFileName); + read(directory, segmentFileName, codecs); return null; } }.run(); @@ -372,6 +381,8 @@ public Object clone() { SegmentInfos sis = (SegmentInfos) super.clone(); for(int i=0;i 0) { - int matchSize = 0; // pop matching terms - match[matchSize++] = (SegmentMergeInfo) queue.pop(); - Term term = match[0].term; - SegmentMergeInfo top = (SegmentMergeInfo) queue.top(); + while (fieldsQueue.size() > 0) { - while (top != null && term.compareTo(top.term) == 0) { - match[matchSize++] = (SegmentMergeInfo) queue.pop(); - top = (SegmentMergeInfo) queue.top(); + while(true) { + SegmentMergeInfo smi = (SegmentMergeInfo) fieldsQueue.pop(); + if (smi.nextTerm()) { + termsQueue.add(smi); + } else if (smi.nextField()) { + // field had no terms + fieldsQueue.add(smi); + } else { + // done with a segment + } + SegmentMergeInfo top = (SegmentMergeInfo) fieldsQueue.top(); + if (top == null || (termsQueue.size() > 0 && ((SegmentMergeInfo) termsQueue.top()).field != top.field)) { + break; + } } + + if (termsQueue.size() > 0) { + // merge one field - if (currentField != term.field) { - currentField = term.field; - if (termsConsumer != null) - termsConsumer.finish(); - final FieldInfo fieldInfo = fieldInfos.fieldInfo(currentField); - termsConsumer = consumer.addField(fieldInfo); + final String field = ((SegmentMergeInfo) termsQueue.top()).field; + if (Codec.DEBUG) { + System.out.println("merge field=" + field + " segCount=" + termsQueue.size()); + } + final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); + final TermsConsumer termsConsumer = consumer.addField(fieldInfo); omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions; - } - int df = appendPostings(termsConsumer, match, matchSize); // add new TermInfo + while(termsQueue.size() > 0) { + // pop matching terms + int matchSize = 0; + while(true) { + match[matchSize++] = (SegmentMergeInfo) termsQueue.pop(); + SegmentMergeInfo top = (SegmentMergeInfo) termsQueue.top(); + if (top == null || !top.term.termEquals(match[0].term)) { + break; + } + } + + if (Codec.DEBUG) { + System.out.println("merge field=" + field + " term=" + match[0].term + " numReaders=" + matchSize); + } + + int df = appendPostings(termsConsumer, match, matchSize); - checkAbort.work(df/3.0); + checkAbort.work(df/3.0); - while (matchSize > 0) { - SegmentMergeInfo smi = match[--matchSize]; - if (smi.next()) - queue.add(smi); // restore queue - else - smi.close(); // done with a segment + // put SegmentMergeInfos back into repsective queues + while (matchSize > 0) { + SegmentMergeInfo smi = match[--matchSize]; + if (smi.nextTerm()) { + termsQueue.add(smi); + } else if (smi.nextField()) { + fieldsQueue.add(smi); + } else { + // done with a segment + } + } + } + termsConsumer.finish(); } } } @@ -668,6 +726,8 @@ int[] getDelCounts() { return delCounts; } + + private final UnicodeUtil.UTF16Result termBuffer = new UnicodeUtil.UTF16Result(); /** Process postings from multiple segments all positioned on the * same term. Writes out merged entries into freqOutput and @@ -679,45 +739,80 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int appendPostings(final FormatPostingsTermsConsumer termsConsumer, SegmentMergeInfo[] smis, int n) + private final int appendPostings(final TermsConsumer termsConsumer, SegmentMergeInfo[] smis, int n) throws CorruptIndexException, IOException { - final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(smis[0].term.text); + // nocommit -- maybe cutover TermsConsumer API to + // TermRef as well? + final TermRef text = smis[0].term; + UnicodeUtil.UTF8toUTF16(text.bytes, text.offset, text.length, termBuffer); + + // Make space for terminator + final int length = termBuffer.length; + termBuffer.setLength(1+termBuffer.length); + + // nocommit -- make this a static final constant somewhere: + termBuffer.result[length] = 0xffff; + + final DocsConsumer docConsumer = termsConsumer.startTerm(termBuffer.result, 0); + int df = 0; for (int i = 0; i < n; i++) { + if (Codec.DEBUG) { + System.out.println(" merge reader " + (i+1) + " of " + n + ": term=" + text); + } + SegmentMergeInfo smi = smis[i]; - TermPositions postings = smi.getPositions(); - assert postings != null; + DocsEnum docs = smi.terms.docs(smi.reader.getDeletedDocs()); int base = smi.base; int[] docMap = smi.getDocMap(); - postings.seek(smi.termEnum); - while (postings.next()) { + while (true) { + int startDoc = docs.next(); + if (startDoc == DocsEnum.NO_MORE_DOCS) { + break; + } + if (Codec.DEBUG) { + System.out.println(" merge read doc=" + startDoc); + } + df++; - int doc = postings.doc(); - if (docMap != null) - doc = docMap[doc]; // map around deletions + int doc; + if (docMap != null) { + // map around deletions + doc = docMap[startDoc]; + assert doc != -1: "postings enum returned deleted docID " + startDoc + " freq=" + docs.freq() + " df=" + df; + } else { + doc = startDoc; + } + doc += base; // convert to merged space + assert doc < mergedDocs: "doc=" + doc + " maxDoc=" + mergedDocs; - final int freq = postings.freq(); - final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(doc, freq); + final int freq = docs.freq(); + final PositionsConsumer posConsumer = docConsumer.addDoc(doc, freq); + final PositionsEnum positions = docs.positions(); + // nocommit -- omitTF should be "private", and this + // code (and FreqProxTermsWriter) should instead + // check if posConsumer is null? + if (!omitTermFreqAndPositions) { for (int j = 0; j < freq; j++) { - final int position = postings.nextPosition(); - final int payloadLength = postings.getPayloadLength(); + final int position = positions.next(); + final int payloadLength = positions.getPayloadLength(); if (payloadLength > 0) { if (payloadBuffer == null || payloadBuffer.length < payloadLength) payloadBuffer = new byte[payloadLength]; - postings.getPayload(payloadBuffer, 0); + positions.getPayload(payloadBuffer, 0); } posConsumer.addPosition(position, payloadBuffer, 0, payloadLength); } - posConsumer.finish(); + posConsumer.finishDoc(); } } } - docConsumer.finish(); + termsConsumer.finishTerm(termBuffer.result, 0, df); return df; } Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 822088) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -36,7 +36,16 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BitVector; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.CloseableThreadLocal; +import org.apache.lucene.util.cache.Cache; +import org.apache.lucene.util.cache.SimpleLRUCache; +import org.apache.lucene.index.codecs.Codecs; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.preflex.PreFlexFields; +import org.apache.lucene.index.codecs.preflex.SegmentTermDocs; +import org.apache.lucene.index.codecs.preflex.SegmentTermPositions; +import org.apache.lucene.index.codecs.FieldsProducer; /** @version $Id */ /** @@ -48,6 +57,7 @@ private SegmentInfo si; private int readBufferSize; + boolean isPreFlex; CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); @@ -83,23 +93,35 @@ final String segment; final FieldInfos fieldInfos; - final IndexInput freqStream; - final IndexInput proxStream; - final TermInfosReader tisNoIndex; + final FieldsProducer fields; + final boolean isPreFlex; + final Codecs codecs; + final Directory dir; final Directory cfsDir; final int readBufferSize; final int termsIndexDivisor; - TermInfosReader tis; FieldsReader fieldsReaderOrig; TermVectorsReader termVectorsReaderOrig; CompoundFileReader cfsReader; CompoundFileReader storeCFSReader; - CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException { + CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, Codecs codecs) throws IOException { + + if (termsIndexDivisor < 1 && termsIndexDivisor != -1) { + throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + termsIndexDivisor); + } + segment = si.name; + if (Codec.DEBUG) { + System.out.println("sr: init core for segment=" + segment); + } + if (codecs == null) { + codecs = Codecs.getDefault(); + } + this.codecs = codecs; this.readBufferSize = readBufferSize; this.dir = dir; @@ -116,23 +138,15 @@ fieldInfos = new FieldInfos(cfsDir, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION); this.termsIndexDivisor = termsIndexDivisor; - TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor); - if (termsIndexDivisor == -1) { - tisNoIndex = reader; - } else { - tis = reader; - tisNoIndex = null; - } - // make sure that all index files have been read or are kept open - // so that if an index update removes them we'll still have them - freqStream = cfsDir.openInput(segment + "." + IndexFileNames.FREQ_EXTENSION, readBufferSize); - - if (fieldInfos.hasProx()) { - proxStream = cfsDir.openInput(segment + "." + IndexFileNames.PROX_EXTENSION, readBufferSize); - } else { - proxStream = null; + // Ask codec for its Fields + if (Codec.DEBUG) { + System.out.println("sr.core.init: seg=" + si.name + " codec=" + si.getCodec()); } + fields = si.getCodec().fieldsProducer(cfsDir, fieldInfos, si, readBufferSize, termsIndexDivisor); + assert fields != null; + + isPreFlex = fields instanceof PreFlexFields; success = true; } finally { if (!success) { @@ -157,64 +171,12 @@ return cfsReader; } - synchronized TermInfosReader getTermsReader() { - if (tis != null) { - return tis; - } else { - return tisNoIndex; - } - } - - synchronized boolean termsIndexIsLoaded() { - return tis != null; - } - - // NOTE: only called from IndexWriter when a near - // real-time reader is opened, or applyDeletes is run, - // sharing a segment that's still being merged. This - // method is not fully thread safe, and relies on the - // synchronization in IndexWriter - synchronized void loadTermsIndex(SegmentInfo si, int termsIndexDivisor) throws IOException { - if (tis == null) { - Directory dir0; - if (si.getUseCompoundFile()) { - // In some cases, we were originally opened when CFS - // was not used, but then we are asked to open the - // terms reader with index, the segment has switched - // to CFS - if (cfsReader == null) { - cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize); - } - dir0 = cfsReader; - } else { - dir0 = dir; - } - - tis = new TermInfosReader(dir0, segment, fieldInfos, readBufferSize, termsIndexDivisor); - } - } - synchronized void decRef() throws IOException { if (ref.decRef() == 0) { - // close everything, nothing is shared anymore with other readers - if (tis != null) { - tis.close(); - // null so if an app hangs on to us we still free most ram - tis = null; - } - - if (tisNoIndex != null) { - tisNoIndex.close(); - } - - if (freqStream != null) { - freqStream.close(); - } - - if (proxStream != null) { - proxStream.close(); + if (fields != null) { + fields.close(); } if (termVectorsReaderOrig != null) { @@ -588,7 +550,7 @@ * @deprecated */ public static SegmentReader get(SegmentInfo si) throws CorruptIndexException, IOException { - return get(false, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + return get(false, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null); } /** @@ -596,7 +558,7 @@ * @throws IOException if there is a low-level IO error */ public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor); + return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor, null); } /** @@ -605,7 +567,7 @@ * @deprecated */ static SegmentReader get(SegmentInfo si, int readBufferSize, boolean doOpenStores, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return get(false, si.dir, si, readBufferSize, doOpenStores, termInfosIndexDivisor); + return get(false, si.dir, si, readBufferSize, doOpenStores, termInfosIndexDivisor, null); } /** @@ -617,8 +579,13 @@ SegmentInfo si, int readBufferSize, boolean doOpenStores, - int termInfosIndexDivisor) + int termInfosIndexDivisor, + Codecs codecs) throws CorruptIndexException, IOException { + if (codecs == null) { + codecs = Codecs.getDefault(); + } + SegmentReader instance; try { if (readOnly) @@ -635,7 +602,7 @@ boolean success = false; try { - instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor); + instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor, codecs); if (doOpenStores) { instance.core.openDocStores(si); } @@ -660,6 +627,10 @@ core.openDocStores(si); } + public synchronized Bits getDeletedDocs() { + return deletedDocs; + } + private void loadDeletedDocs() throws IOException { // NOTE: the bitvector is stored using the regular directory, not cfs if (hasDeletions(si)) { @@ -929,14 +900,32 @@ return new ArrayList(si.files()); } - public TermEnum terms() { + public TermEnum terms() throws IOException { ensureOpen(); - return core.getTermsReader().terms(); + if (isPreFlex) { + // For old API on an old segment, instead of + // converting old API -> new API -> old API, just give + // direct access to old: + return ((PreFlexFields) core.fields).tis.terms(); + } else { + // Emulate old API on top of new index + return new LegacyTermEnum(null); + } } + /** @deprecated Please switch to the flex API ({@link + * #fields}) instead. */ public TermEnum terms(Term t) throws IOException { ensureOpen(); - return core.getTermsReader().terms(t); + if (isPreFlex) { + // For old API on an old segment, instead of + // converting old API -> new API -> old API, just give + // direct access to old: + return ((PreFlexFields) core.fields).tis.terms(t); + } else { + // Emulate old API on top of new index + return new LegacyTermEnum(t); + } } FieldInfos fieldInfos() { @@ -952,6 +941,8 @@ return (deletedDocs != null && deletedDocs.get(n)); } + /** @deprecated Switch to the flex API ({@link + * IndexReader#termDocsEnum}) instead. */ public TermDocs termDocs(Term term) throws IOException { if (term == null) { return new AllTermDocs(this); @@ -959,26 +950,88 @@ return super.termDocs(term); } } + + public Fields fields() throws IOException { + return core.fields; + } + /** @deprecated Switch to the flex API {@link + * IndexReader#termDocsEnum} instead. */ public TermDocs termDocs() throws IOException { ensureOpen(); - return new SegmentTermDocs(this); + if (isPreFlex) { + // For old API on an old segment, instead of + // converting old API -> new API -> old API, just give + // direct access to old: + final PreFlexFields pre = (PreFlexFields) core.fields; + return new SegmentTermDocs(pre.freqStream, deletedDocs, pre.tis, core.fieldInfos); + } else { + // Emulate old API + return new LegacyTermDocs(); + } } + /** @deprecated Switch to the flex API {@link + * IndexReader#termDocsEnum} instead */ public TermPositions termPositions() throws IOException { ensureOpen(); - return new SegmentTermPositions(this); + if (isPreFlex) { + // For old API on an old segment, instead of + // converting old API -> new API -> old API, just give + // direct access to old: + final PreFlexFields pre = (PreFlexFields) core.fields; + return new SegmentTermPositions(pre.freqStream, pre.proxStream, deletedDocs, pre.tis, core.fieldInfos); + } else + // Emulate old API + return new LegacyTermPositions(); } + private final CloseableThreadLocal perThread = new CloseableThreadLocal(); + + // nocommit -- move term vectors under here + private static final class PerThread { + LegacyTermEnum terms; + + // Used for caching the least recently looked-up Terms + Cache termsCache; + } + + private final static int DEFAULT_TERMS_CACHE_SIZE = 1024; + + private PerThread getPerThread() throws IOException { + PerThread resources = (PerThread) perThread.get(); + if (resources == null) { + resources = new PerThread(); + resources.terms = new LegacyTermEnum(null); + // Cache does not have to be thread-safe, it is only used by one thread at the same time + resources.termsCache = new SimpleLRUCache(DEFAULT_TERMS_CACHE_SIZE); + perThread.set(resources); + } + return resources; + } + + public int docFreq(Term t) throws IOException { ensureOpen(); - TermInfo ti = core.getTermsReader().get(t); - if (ti != null) - return ti.docFreq; - else + Terms terms = core.fields.terms(t.field); + if (terms != null) { + return terms.docFreq(new TermRef(t.text)); + } else { return 0; + } } + public int docFreq(String field, TermRef term) throws IOException { + ensureOpen(); + + Terms terms = core.fields.terms(field); + if (terms != null) { + return terms.docFreq(term); + } else { + return 0; + } + } + public int numDocs() { // Don't call ensureOpen() here (it could affect performance) int n = maxDoc(); @@ -1146,17 +1199,13 @@ } } - boolean termsIndexLoaded() { - return core.termsIndexIsLoaded(); - } - // NOTE: only called from IndexWriter when a near // real-time reader is opened, or applyDeletes is run, // sharing a segment that's still being merged. This // method is not thread safe, and relies on the // synchronization in IndexWriter - void loadTermsIndex(int termsIndexDivisor) throws IOException { - core.loadTermsIndex(si, termsIndexDivisor); + void loadTermsIndex() throws IOException { + core.fields.loadTermsIndex(); } // for testing only @@ -1323,12 +1372,9 @@ // This is necessary so that cloned SegmentReaders (which // share the underlying postings data) will map to the // same entry in the FieldCache. See LUCENE-1579. + // nocommit - what to return here? public final Object getFieldCacheKey() { - return core.freqStream; - } - - public long getUniqueTermCount() { - return core.getTermsReader().size(); + return core; } /** @@ -1339,7 +1385,7 @@ * @deprecated Remove this when tests are fixed! */ static SegmentReader getOnlySegmentReader(Directory dir) throws IOException { - return getOnlySegmentReader(IndexReader.open(dir,false)); + return getOnlySegmentReader(IndexReader.open(dir, false)); } static SegmentReader getOnlySegmentReader(IndexReader reader) { @@ -1360,4 +1406,255 @@ public int getTermInfosIndexDivisor() { return core.termsIndexDivisor; } + + // Back compat: legacy TermEnum API over flex API + final private class LegacyTermEnum extends TermEnum { + FieldsEnum fields; + TermsEnum terms; + boolean done; + String currentField; + TermRef currentTerm; + + public LegacyTermEnum(Term t) throws IOException { + //System.out.println("sr.lte.init: term=" + t); + fields = core.fields.iterator(); + currentField = fields.next(); + if (currentField == null) { + done = true; + } else if (t != null) { + // Pre-seek + + // nocommit -- inefficient; do we need + // FieldsEnum.seek? (but this is slow only for + // legacy API, and, when field count is high) + while(currentField.compareTo(t.field) < 0) { + currentField = fields.next(); + if (currentField == null) { + // Didn't find the field + done = true; + break; + } + } + + if (!done) { + if (currentField == t.field) { + // Field matches -- get terms + terms = fields.terms(); + TermsEnum.SeekResult result = terms.seek(new TermRef(t.text())); + if (result.status == TermsEnum.SeekResult.Status.END) { + // leave currentTerm null + } else if (result.status == TermsEnum.SeekResult.Status.FOUND) { + currentTerm = new TermRef(t.text()); + } else { + currentTerm = result.term; + } + // System.out.println("done seek"); + } + } + } else { + terms = fields.terms(); + } + } + + public boolean next() throws IOException { + + if (Codec.DEBUG) { + System.out.println("tdte.next done=" + done + " seg=" + core.segment); + } + + if (done) { + return false; + } + + while(true) { + if (terms == null) { + // Advance to the next field + currentField = fields.next(); + if (currentField == null) { + if (Codec.DEBUG) + System.out.println(" fields.next returned false"); + done = true; + return false; + } + terms = fields.terms(); + } + currentTerm = terms.next(); + if (currentTerm != null) { + // This field still has terms + return true; + } else { + // Done producing terms from this field + terms = null; + } + } + } + + public Term term() { + if (terms != null && !done) { + if (currentTerm != null) { + return new Term(currentField, currentTerm.toString()); + } + } + return null; + } + + public int docFreq() { + return terms == null ? 0 : terms.docFreq(); + } + + public void close() {} + } + + // Back compat: emulates legacy TermDocs API on top of + // flex API + private class LegacyTermDocs implements TermDocs { + + String currentField; + final Fields fields; + TermsEnum terms; + DocsEnum docs; + int doc; + + LegacyTermDocs() throws IOException { + fields = core.fields; + } + + public void close() {} + + public void seek(TermEnum termEnum) throws IOException { + // nocommit -- optimize for the special cases here + seek(termEnum.term()); + } + + public boolean skipTo(int target) throws IOException { + if (docs == null) return false; + doc = docs.advance(target); + return doc != docs.NO_MORE_DOCS; + } + + public int read(int[] docs, int[] freqs) throws IOException { + if (this.docs == null) { + return 0; + } + return this.docs.read(docs, freqs); + } + + public void seek(Term term) throws IOException { + + if (Codec.DEBUG) { + System.out.println("\nwrapper termdocs.seek term=" + term); + } + + docs = null; + + if (terms != null && !term.field.equals(currentField)) { + if (Codec.DEBUG) { + System.out.println(" switch field"); + } + if (terms != null) { + terms = null; + } + } + + if (terms == null) { + currentField = term.field; + Terms terms1 = fields.terms(term.field); + if (terms1 == null) { + // no such field + return; + } else { + terms = terms1.iterator(); + } + } + + TermsEnum.SeekResult result = terms.seek(new TermRef(term.text)); + if (result.status == TermsEnum.SeekResult.Status.FOUND) { + // Term exists + docs = terms.docs(deletedDocs); + if (Codec.DEBUG) { + System.out.println(" init docs enum"); + } + } else { + docs = null; + if (Codec.DEBUG) { + System.out.println(" clear docs enum"); + } + } + } + + public int doc() { + if (docs == null) return 0; + else return doc; + } + + public int freq() { + if (docs == null) return 0; + return docs.freq(); + } + + public boolean next() throws IOException { + if (docs == null) return false; + doc = docs.next(); + return doc != DocsEnum.NO_MORE_DOCS; + } + } + + // Back compat: implements legacy TermPositions API on top + // of flex API + final private class LegacyTermPositions extends LegacyTermDocs implements TermPositions { + + PositionsEnum positions; + + LegacyTermPositions() throws IOException { + super(); + } + + public void seek(TermEnum termEnum) throws IOException { + super.seek(termEnum); + if (docs != null) + positions = docs.positions(); + } + + public boolean skipTo(int target) throws IOException { + boolean result = super.skipTo(target); + positions = null; + return result; + } + + public int read(int[] docs, int[] freqs) throws IOException { + throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead."); + } + + public void seek(Term term) throws IOException { + super.seek(term); + positions = null; + } + + public boolean next() throws IOException { + boolean result = super.next(); + positions = null; + return result; + } + + public int nextPosition() throws IOException { + if (positions == null) { + positions = docs.positions(); + } + return positions.next(); + } + + public int getPayloadLength() { + return positions.getPayloadLength(); + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + return positions.getPayload(data, offset); + } + + public boolean isPayloadAvailable() { + return positions.hasPayload(); + } + } + + } Index: src/java/org/apache/lucene/index/SegmentTermDocs.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermDocs.java (revision 822088) +++ src/java/org/apache/lucene/index/SegmentTermDocs.java (working copy) @@ -1,212 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import org.apache.lucene.util.BitVector; -import org.apache.lucene.store.IndexInput; - -class SegmentTermDocs implements TermDocs { - protected SegmentReader parent; - protected IndexInput freqStream; - protected int count; - protected int df; - protected BitVector deletedDocs; - int doc = 0; - int freq; - - private int skipInterval; - private int maxSkipLevels; - private DefaultSkipListReader skipListReader; - - private long freqBasePointer; - private long proxBasePointer; - - private long skipPointer; - private boolean haveSkipped; - - protected boolean currentFieldStoresPayloads; - protected boolean currentFieldOmitTermFreqAndPositions; - - protected SegmentTermDocs(SegmentReader parent) { - this.parent = parent; - this.freqStream = (IndexInput) parent.core.freqStream.clone(); - synchronized (parent) { - this.deletedDocs = parent.deletedDocs; - } - this.skipInterval = parent.core.getTermsReader().getSkipInterval(); - this.maxSkipLevels = parent.core.getTermsReader().getMaxSkipLevels(); - } - - public void seek(Term term) throws IOException { - TermInfo ti = parent.core.getTermsReader().get(term); - seek(ti, term); - } - - public void seek(TermEnum termEnum) throws IOException { - TermInfo ti; - Term term; - - // use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs - if (termEnum instanceof SegmentTermEnum && ((SegmentTermEnum) termEnum).fieldInfos == parent.core.fieldInfos) { // optimized case - SegmentTermEnum segmentTermEnum = ((SegmentTermEnum) termEnum); - term = segmentTermEnum.term(); - ti = segmentTermEnum.termInfo(); - } else { // punt case - term = termEnum.term(); - ti = parent.core.getTermsReader().get(term); - } - - seek(ti, term); - } - - void seek(TermInfo ti, Term term) throws IOException { - count = 0; - FieldInfo fi = parent.core.fieldInfos.fieldInfo(term.field); - currentFieldOmitTermFreqAndPositions = (fi != null) ? fi.omitTermFreqAndPositions : false; - currentFieldStoresPayloads = (fi != null) ? fi.storePayloads : false; - if (ti == null) { - df = 0; - } else { - df = ti.docFreq; - doc = 0; - freqBasePointer = ti.freqPointer; - proxBasePointer = ti.proxPointer; - skipPointer = freqBasePointer + ti.skipOffset; - freqStream.seek(freqBasePointer); - haveSkipped = false; - } - } - - public void close() throws IOException { - freqStream.close(); - if (skipListReader != null) - skipListReader.close(); - } - - public final int doc() { return doc; } - public final int freq() { return freq; } - - protected void skippingDoc() throws IOException { - } - - public boolean next() throws IOException { - while (true) { - if (count == df) - return false; - final int docCode = freqStream.readVInt(); - - if (currentFieldOmitTermFreqAndPositions) { - doc += docCode; - freq = 1; - } else { - doc += docCode >>> 1; // shift off low bit - if ((docCode & 1) != 0) // if low bit is set - freq = 1; // freq is one - else - freq = freqStream.readVInt(); // else read freq - } - - count++; - - if (deletedDocs == null || !deletedDocs.get(doc)) - break; - skippingDoc(); - } - return true; - } - - /** Optimized implementation. */ - public int read(final int[] docs, final int[] freqs) - throws IOException { - final int length = docs.length; - if (currentFieldOmitTermFreqAndPositions) { - return readNoTf(docs, freqs, length); - } else { - int i = 0; - while (i < length && count < df) { - // manually inlined call to next() for speed - final int docCode = freqStream.readVInt(); - doc += docCode >>> 1; // shift off low bit - if ((docCode & 1) != 0) // if low bit is set - freq = 1; // freq is one - else - freq = freqStream.readVInt(); // else read freq - count++; - - if (deletedDocs == null || !deletedDocs.get(doc)) { - docs[i] = doc; - freqs[i] = freq; - ++i; - } - } - return i; - } - } - - private final int readNoTf(final int[] docs, final int[] freqs, final int length) throws IOException { - int i = 0; - while (i < length && count < df) { - // manually inlined call to next() for speed - doc += freqStream.readVInt(); - count++; - - if (deletedDocs == null || !deletedDocs.get(doc)) { - docs[i] = doc; - // Hardware freq to 1 when term freqs were not - // stored in the index - freqs[i] = 1; - ++i; - } - } - return i; - } - - - /** Overridden by SegmentTermPositions to skip in prox stream. */ - protected void skipProx(long proxPointer, int payloadLength) throws IOException {} - - /** Optimized implementation. */ - public boolean skipTo(int target) throws IOException { - if (df >= skipInterval) { // optimized case - if (skipListReader == null) - skipListReader = new DefaultSkipListReader((IndexInput) freqStream.clone(), maxSkipLevels, skipInterval); // lazily clone - - if (!haveSkipped) { // lazily initialize skip stream - skipListReader.init(skipPointer, freqBasePointer, proxBasePointer, df, currentFieldStoresPayloads); - haveSkipped = true; - } - - int newCount = skipListReader.skipTo(target); - if (newCount > count) { - freqStream.seek(skipListReader.getFreqPointer()); - skipProx(skipListReader.getProxPointer(), skipListReader.getPayloadLength()); - - doc = skipListReader.getDoc(); - count = newCount; - } - } - - // done skipping, now just scan - do { - if (!next()) - return false; - } while (target > doc); - return true; - } -} Index: src/java/org/apache/lucene/index/SegmentTermEnum.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermEnum.java (revision 822088) +++ src/java/org/apache/lucene/index/SegmentTermEnum.java (working copy) @@ -1,211 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import org.apache.lucene.store.IndexInput; - -final class SegmentTermEnum extends TermEnum implements Cloneable { - private IndexInput input; - FieldInfos fieldInfos; - long size; - long position = -1; - - private TermBuffer termBuffer = new TermBuffer(); - private TermBuffer prevBuffer = new TermBuffer(); - private TermBuffer scanBuffer = new TermBuffer(); // used for scanning - - private TermInfo termInfo = new TermInfo(); - - private int format; - private boolean isIndex = false; - long indexPointer = 0; - int indexInterval; - int skipInterval; - int maxSkipLevels; - private int formatM1SkipInterval; - - SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi) - throws CorruptIndexException, IOException { - input = i; - fieldInfos = fis; - isIndex = isi; - maxSkipLevels = 1; // use single-level skip lists for formats > -3 - - int firstInt = input.readInt(); - if (firstInt >= 0) { - // original-format file, without explicit format version number - format = 0; - size = firstInt; - - // back-compatible settings - indexInterval = 128; - skipInterval = Integer.MAX_VALUE; // switch off skipTo optimization - } else { - // we have a format version number - format = firstInt; - - // check that it is a format we can understand - if (format < TermInfosWriter.FORMAT_CURRENT) - throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher"); - - size = input.readLong(); // read the size - - if(format == -1){ - if (!isIndex) { - indexInterval = input.readInt(); - formatM1SkipInterval = input.readInt(); - } - // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in - // skipTo implementation of these versions - skipInterval = Integer.MAX_VALUE; - } else { - indexInterval = input.readInt(); - skipInterval = input.readInt(); - if (format <= TermInfosWriter.FORMAT) { - // this new format introduces multi-level skipping - maxSkipLevels = input.readInt(); - } - } - assert indexInterval > 0: "indexInterval=" + indexInterval + " is negative; must be > 0"; - assert skipInterval > 0: "skipInterval=" + skipInterval + " is negative; must be > 0"; - } - if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { - termBuffer.setPreUTF8Strings(); - scanBuffer.setPreUTF8Strings(); - prevBuffer.setPreUTF8Strings(); - } - } - - protected Object clone() { - SegmentTermEnum clone = null; - try { - clone = (SegmentTermEnum) super.clone(); - } catch (CloneNotSupportedException e) {} - - clone.input = (IndexInput) input.clone(); - clone.termInfo = new TermInfo(termInfo); - - clone.termBuffer = (TermBuffer)termBuffer.clone(); - clone.prevBuffer = (TermBuffer)prevBuffer.clone(); - clone.scanBuffer = new TermBuffer(); - - return clone; - } - - final void seek(long pointer, int p, Term t, TermInfo ti) - throws IOException { - input.seek(pointer); - position = p; - termBuffer.set(t); - prevBuffer.reset(); - termInfo.set(ti); - } - - /** Increments the enumeration to the next element. True if one exists.*/ - public final boolean next() throws IOException { - if (position++ >= size - 1) { - prevBuffer.set(termBuffer); - termBuffer.reset(); - return false; - } - - prevBuffer.set(termBuffer); - termBuffer.read(input, fieldInfos); - - termInfo.docFreq = input.readVInt(); // read doc freq - termInfo.freqPointer += input.readVLong(); // read freq pointer - termInfo.proxPointer += input.readVLong(); // read prox pointer - - if(format == -1){ - // just read skipOffset in order to increment file pointer; - // value is never used since skipTo is switched off - if (!isIndex) { - if (termInfo.docFreq > formatM1SkipInterval) { - termInfo.skipOffset = input.readVInt(); - } - } - } - else{ - if (termInfo.docFreq >= skipInterval) - termInfo.skipOffset = input.readVInt(); - } - - if (isIndex) - indexPointer += input.readVLong(); // read index pointer - - return true; - } - - /** Optimized scan, without allocating new terms. - * Return number of invocations to next(). */ - final int scanTo(Term term) throws IOException { - scanBuffer.set(term); - int count = 0; - while (scanBuffer.compareTo(termBuffer) > 0 && next()) { - count++; - } - return count; - } - - /** Returns the current Term in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final Term term() { - return termBuffer.toTerm(); - } - - /** Returns the previous Term enumerated. Initially null.*/ - final Term prev() { - return prevBuffer.toTerm(); - } - - /** Returns the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final TermInfo termInfo() { - return new TermInfo(termInfo); - } - - /** Sets the argument to the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final void termInfo(TermInfo ti) { - ti.set(termInfo); - } - - /** Returns the docFreq from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final int docFreq() { - return termInfo.docFreq; - } - - /* Returns the freqPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long freqPointer() { - return termInfo.freqPointer; - } - - /* Returns the proxPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long proxPointer() { - return termInfo.proxPointer; - } - - /** Closes the enumeration to further activity, freeing resources. */ - public final void close() throws IOException { - input.close(); - } -} Index: src/java/org/apache/lucene/index/SegmentTermPositions.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermPositions.java (revision 822088) +++ src/java/org/apache/lucene/index/SegmentTermPositions.java (working copy) @@ -1,197 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.store.IndexInput; - -import java.io.IOException; - -final class SegmentTermPositions -extends SegmentTermDocs implements TermPositions { - private IndexInput proxStream; - private int proxCount; - private int position; - - // the current payload length - private int payloadLength; - // indicates whether the payload of the current position has - // been read from the proxStream yet - private boolean needToLoadPayload; - - // these variables are being used to remember information - // for a lazy skip - private long lazySkipPointer = -1; - private int lazySkipProxCount = 0; - - SegmentTermPositions(SegmentReader p) { - super(p); - this.proxStream = null; // the proxStream will be cloned lazily when nextPosition() is called for the first time - } - - final void seek(TermInfo ti, Term term) throws IOException { - super.seek(ti, term); - if (ti != null) - lazySkipPointer = ti.proxPointer; - - lazySkipProxCount = 0; - proxCount = 0; - payloadLength = 0; - needToLoadPayload = false; - } - - public final void close() throws IOException { - super.close(); - if (proxStream != null) proxStream.close(); - } - - public final int nextPosition() throws IOException { - if (currentFieldOmitTermFreqAndPositions) - // This field does not store term freq, positions, payloads - return 0; - // perform lazy skips if necessary - lazySkip(); - proxCount--; - return position += readDeltaPosition(); - } - - private final int readDeltaPosition() throws IOException { - int delta = proxStream.readVInt(); - if (currentFieldStoresPayloads) { - // if the current field stores payloads then - // the position delta is shifted one bit to the left. - // if the LSB is set, then we have to read the current - // payload length - if ((delta & 1) != 0) { - payloadLength = proxStream.readVInt(); - } - delta >>>= 1; - needToLoadPayload = true; - } - return delta; - } - - protected final void skippingDoc() throws IOException { - // we remember to skip a document lazily - lazySkipProxCount += freq; - } - - public final boolean next() throws IOException { - // we remember to skip the remaining positions of the current - // document lazily - lazySkipProxCount += proxCount; - - if (super.next()) { // run super - proxCount = freq; // note frequency - position = 0; // reset position - return true; - } - return false; - } - - public final int read(final int[] docs, final int[] freqs) { - throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead."); - } - - - /** Called by super.skipTo(). */ - protected void skipProx(long proxPointer, int payloadLength) throws IOException { - // we save the pointer, we might have to skip there lazily - lazySkipPointer = proxPointer; - lazySkipProxCount = 0; - proxCount = 0; - this.payloadLength = payloadLength; - needToLoadPayload = false; - } - - private void skipPositions(int n) throws IOException { - assert !currentFieldOmitTermFreqAndPositions; - for (int f = n; f > 0; f--) { // skip unread positions - readDeltaPosition(); - skipPayload(); - } - } - - private void skipPayload() throws IOException { - if (needToLoadPayload && payloadLength > 0) { - proxStream.seek(proxStream.getFilePointer() + payloadLength); - } - needToLoadPayload = false; - } - - // It is not always necessary to move the prox pointer - // to a new document after the freq pointer has been moved. - // Consider for example a phrase query with two terms: - // the freq pointer for term 1 has to move to document x - // to answer the question if the term occurs in that document. But - // only if term 2 also matches document x, the positions have to be - // read to figure out if term 1 and term 2 appear next - // to each other in document x and thus satisfy the query. - // So we move the prox pointer lazily to the document - // as soon as positions are requested. - private void lazySkip() throws IOException { - if (proxStream == null) { - // clone lazily - proxStream = (IndexInput) parent.core.proxStream.clone(); - } - - // we might have to skip the current payload - // if it was not read yet - skipPayload(); - - if (lazySkipPointer != -1) { - proxStream.seek(lazySkipPointer); - lazySkipPointer = -1; - } - - if (lazySkipProxCount != 0) { - skipPositions(lazySkipProxCount); - lazySkipProxCount = 0; - } - } - - public int getPayloadLength() { - return payloadLength; - } - - public byte[] getPayload(byte[] data, int offset) throws IOException { - if (!needToLoadPayload) { - throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); - } - - // read payloads lazily - byte[] retArray; - int retOffset; - if (data == null || data.length - offset < payloadLength) { - // the array is too small to store the payload data, - // so we allocate a new one - retArray = new byte[payloadLength]; - retOffset = 0; - } else { - retArray = data; - retOffset = offset; - } - proxStream.readBytes(retArray, retOffset, payloadLength); - needToLoadPayload = false; - return retArray; - } - - public boolean isPayloadAvailable() { - return needToLoadPayload && payloadLength > 0; - } - -} Index: src/java/org/apache/lucene/index/SegmentWriteState.java =================================================================== --- src/java/org/apache/lucene/index/SegmentWriteState.java (revision 808825) +++ src/java/org/apache/lucene/index/SegmentWriteState.java (working copy) @@ -1,50 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.HashSet; -import java.util.Collection; - -import org.apache.lucene.store.Directory; - -class SegmentWriteState { - DocumentsWriter docWriter; - Directory directory; - String segmentName; - String docStoreSegmentName; - int numDocs; - int termIndexInterval; - int numDocsInStore; - Collection flushedFiles; - - public SegmentWriteState(DocumentsWriter docWriter, Directory directory, String segmentName, String docStoreSegmentName, int numDocs, - int numDocsInStore, int termIndexInterval) { - this.docWriter = docWriter; - this.directory = directory; - this.segmentName = segmentName; - this.docStoreSegmentName = docStoreSegmentName; - this.numDocs = numDocs; - this.numDocsInStore = numDocsInStore; - this.termIndexInterval = termIndexInterval; - flushedFiles = new HashSet(); - } - - public String segmentFileName(String ext) { - return segmentName + "." + ext; - } -} Index: src/java/org/apache/lucene/index/SegmentWriteState.java =================================================================== --- src/java/org/apache/lucene/index/SegmentWriteState.java (revision 0) +++ src/java/org/apache/lucene/index/SegmentWriteState.java (revision 808825) @@ -0,0 +1,91 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.HashSet; +import java.util.Collection; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.Codecs; + +/** + * This class is not meant for public usage; it's only + * public in order to expose access across packages. It's + * used internally when updating the index. + */ +public class SegmentWriteState { + DocumentsWriter docWriter; + // nocommit -- made public + public Directory directory; + // nocommit -- made public + public String segmentName; + // nocommit -- made public + public FieldInfos fieldInfos; + String docStoreSegmentName; + // nocommit -- made public + public int numDocs; + int numDocsInStore; + // nocommit -- made public + public Collection flushedFiles; + + // Actual codec used + Codec codec; + + /** Expert: The fraction of terms in the "dictionary" which should be stored + * in RAM. Smaller values use more memory, but make searching slightly + * faster, while larger values use less memory and make searching slightly + * slower. Searching is typically not dominated by dictionary lookup, so + * tweaking this is rarely useful.*/ + // nocommit -- made public + public int termIndexInterval; + + /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, + * used to accelerate {@link TermDocs#skipTo(int)}. Larger values result in + * smaller indexes, greater acceleration, but fewer accelerable cases, while + * smaller values result in bigger indexes, less acceleration and more + * accelerable cases. More detailed experiments would be useful here. */ + // nocommit -- made public + public int skipInterval = 16; + + /** Expert: The maximum number of skip levels. Smaller values result in + * slightly smaller indexes, but slower skipping in big posting lists. + */ + // nocommit -- made public + public int maxSkipLevels = 10; + + public SegmentWriteState(DocumentsWriter docWriter, Directory directory, String segmentName, FieldInfos fieldInfos, + String docStoreSegmentName, int numDocs, + int numDocsInStore, int termIndexInterval, + Codecs codecs) { + this.docWriter = docWriter; + this.directory = directory; + this.segmentName = segmentName; + this.fieldInfos = fieldInfos; + this.docStoreSegmentName = docStoreSegmentName; + this.numDocs = numDocs; + this.numDocsInStore = numDocsInStore; + this.termIndexInterval = termIndexInterval; + this.codec = codecs.getWriter(this); + flushedFiles = new HashSet(); + } + + public String segmentFileName(String ext) { + return segmentName + "." + ext; + } +} Index: src/java/org/apache/lucene/index/Term.java =================================================================== --- src/java/org/apache/lucene/index/Term.java (revision 822088) +++ src/java/org/apache/lucene/index/Term.java (working copy) @@ -1,7 +1,5 @@ package org.apache.lucene.index; -import org.apache.lucene.util.StringHelper; - /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -19,6 +17,8 @@ * limitations under the License. */ +import org.apache.lucene.util.StringHelper; + /** A Term represents a word from text. This is the unit of search. It is composed of two elements, the text of the word, as a string, and the name of @@ -49,7 +49,8 @@ this(fld, "", true); } - Term(String fld, String txt, boolean intern) { + // nocommit -- made public + public Term(String fld, String txt, boolean intern) { field = intern ? StringHelper.intern(fld) : fld; // field names are interned text = txt; // unless already known to be } Index: src/java/org/apache/lucene/index/TermBuffer.java =================================================================== --- src/java/org/apache/lucene/index/TermBuffer.java (revision 822088) +++ src/java/org/apache/lucene/index/TermBuffer.java (working copy) @@ -1,139 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.UnicodeUtil; - -final class TermBuffer implements Cloneable { - - private String field; - private Term term; // cached - private boolean preUTF8Strings; // true if strings are stored in modified UTF8 encoding (LUCENE-510) - private boolean dirty; // true if text was set externally (ie not read via UTF8 bytes) - - private UnicodeUtil.UTF16Result text = new UnicodeUtil.UTF16Result(); - private UnicodeUtil.UTF8Result bytes = new UnicodeUtil.UTF8Result(); - - public final int compareTo(TermBuffer other) { - if (field == other.field) // fields are interned - return compareChars(text.result, text.length, other.text.result, other.text.length); - else - return field.compareTo(other.field); - } - - private static final int compareChars(char[] chars1, int len1, - char[] chars2, int len2) { - final int end = len1 < len2 ? len1:len2; - for (int k = 0; k < end; k++) { - char c1 = chars1[k]; - char c2 = chars2[k]; - if (c1 != c2) { - return c1 - c2; - } - } - return len1 - len2; - } - - /** Call this if the IndexInput passed to {@link #read} - * stores terms in the "modified UTF8" (pre LUCENE-510) - * format. */ - void setPreUTF8Strings() { - preUTF8Strings = true; - } - - public final void read(IndexInput input, FieldInfos fieldInfos) - throws IOException { - this.term = null; // invalidate cache - int start = input.readVInt(); - int length = input.readVInt(); - int totalLength = start + length; - if (preUTF8Strings) { - text.setLength(totalLength); - input.readChars(text.result, start, length); - } else { - - if (dirty) { - // Fully convert all bytes since bytes is dirty - UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes); - bytes.setLength(totalLength); - input.readBytes(bytes.result, start, length); - UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text); - dirty = false; - } else { - // Incrementally convert only the UTF8 bytes that are new: - bytes.setLength(totalLength); - input.readBytes(bytes.result, start, length); - UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text); - } - } - this.field = fieldInfos.fieldName(input.readVInt()); - } - - public final void set(Term term) { - if (term == null) { - reset(); - return; - } - final String termText = term.text(); - final int termLen = termText.length(); - text.setLength(termLen); - termText.getChars(0, termLen, text.result, 0); - dirty = true; - field = term.field(); - this.term = term; - } - - public final void set(TermBuffer other) { - text.copyText(other.text); - dirty = true; - field = other.field; - term = other.term; - } - - public void reset() { - field = null; - text.setLength(0); - term = null; - dirty = true; - } - - public Term toTerm() { - if (field == null) // unset - return null; - - if (term == null) - term = new Term(field, new String(text.result, 0, text.length), false); - - return term; - } - - protected Object clone() { - TermBuffer clone = null; - try { - clone = (TermBuffer)super.clone(); - } catch (CloneNotSupportedException e) {} - - clone.dirty = true; - clone.bytes = new UnicodeUtil.UTF8Result(); - clone.text = new UnicodeUtil.UTF16Result(); - clone.text.copyText(text); - return clone; - } -} Index: src/java/org/apache/lucene/index/TermDocs.java =================================================================== --- src/java/org/apache/lucene/index/TermDocs.java (revision 822088) +++ src/java/org/apache/lucene/index/TermDocs.java (working copy) @@ -26,7 +26,8 @@ ordered by document number. @see IndexReader#termDocs() - */ + @deprecated Use {@link DocsEnum} instead +*/ public interface TermDocs { /** Sets this to the data for a term. Index: src/java/org/apache/lucene/index/TermEnum.java =================================================================== --- src/java/org/apache/lucene/index/TermEnum.java (revision 822088) +++ src/java/org/apache/lucene/index/TermEnum.java (working copy) @@ -22,7 +22,8 @@ /** Abstract class for enumerating terms.

Term enumerations are always ordered by Term.compareTo(). Each term in - the enumeration is greater than all that precede it. */ + the enumeration is greater than all that precede it. +* @deprecated Use TermsEnum instead */ public abstract class TermEnum { /** Increments the enumeration to the next element. True if one exists.*/ Index: src/java/org/apache/lucene/index/TermInfo.java =================================================================== --- src/java/org/apache/lucene/index/TermInfo.java (revision 822088) +++ src/java/org/apache/lucene/index/TermInfo.java (working copy) @@ -1,59 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** A TermInfo is the record of information stored for a term.*/ - -final class TermInfo { - /** The number of documents which contain the term. */ - int docFreq = 0; - - long freqPointer = 0; - long proxPointer = 0; - int skipOffset; - - TermInfo() {} - - TermInfo(int df, long fp, long pp) { - docFreq = df; - freqPointer = fp; - proxPointer = pp; - } - - TermInfo(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } - - final void set(int docFreq, - long freqPointer, long proxPointer, int skipOffset) { - this.docFreq = docFreq; - this.freqPointer = freqPointer; - this.proxPointer = proxPointer; - this.skipOffset = skipOffset; - } - - final void set(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } -} Index: src/java/org/apache/lucene/index/TermInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/TermInfosReader.java (revision 822088) +++ src/java/org/apache/lucene/index/TermInfosReader.java (working copy) @@ -1,302 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.cache.Cache; -import org.apache.lucene.util.cache.SimpleLRUCache; -import org.apache.lucene.util.CloseableThreadLocal; - -/** This stores a monotonically increasing set of pairs in a - * Directory. Pairs are accessed either by Term or by ordinal position the - * set. */ - -final class TermInfosReader { - private final Directory directory; - private final String segment; - private final FieldInfos fieldInfos; - - private final CloseableThreadLocal threadResources = new CloseableThreadLocal(); - private final SegmentTermEnum origEnum; - private final long size; - - private final Term[] indexTerms; - private final TermInfo[] indexInfos; - private final long[] indexPointers; - - private final int totalIndexInterval; - - private final static int DEFAULT_CACHE_SIZE = 1024; - - /** - * Per-thread resources managed by ThreadLocal - */ - private static final class ThreadResources { - SegmentTermEnum termEnum; - - // Used for caching the least recently looked-up Terms - Cache termInfoCache; - } - - TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize, int indexDivisor) - throws CorruptIndexException, IOException { - boolean success = false; - - if (indexDivisor < 1 && indexDivisor != -1) { - throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + indexDivisor); - } - - try { - directory = dir; - segment = seg; - fieldInfos = fis; - - origEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_EXTENSION, - readBufferSize), fieldInfos, false); - size = origEnum.size; - - - if (indexDivisor != -1) { - // Load terms index - totalIndexInterval = origEnum.indexInterval * indexDivisor; - final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, - readBufferSize), fieldInfos, true); - - try { - int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index - - indexTerms = new Term[indexSize]; - indexInfos = new TermInfo[indexSize]; - indexPointers = new long[indexSize]; - - for (int i = 0; indexEnum.next(); i++) { - indexTerms[i] = indexEnum.term(); - indexInfos[i] = indexEnum.termInfo(); - indexPointers[i] = indexEnum.indexPointer; - - for (int j = 1; j < indexDivisor; j++) - if (!indexEnum.next()) - break; - } - } finally { - indexEnum.close(); - } - } else { - // Do not load terms index: - totalIndexInterval = -1; - indexTerms = null; - indexInfos = null; - indexPointers = null; - } - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - } - - public int getSkipInterval() { - return origEnum.skipInterval; - } - - public int getMaxSkipLevels() { - return origEnum.maxSkipLevels; - } - - final void close() throws IOException { - if (origEnum != null) - origEnum.close(); - threadResources.close(); - } - - /** Returns the number of term/value pairs in the set. */ - final long size() { - return size; - } - - private ThreadResources getThreadResources() { - ThreadResources resources = (ThreadResources)threadResources.get(); - if (resources == null) { - resources = new ThreadResources(); - resources.termEnum = terms(); - // Cache does not have to be thread-safe, it is only used by one thread at the same time - resources.termInfoCache = new SimpleLRUCache(DEFAULT_CACHE_SIZE); - threadResources.set(resources); - } - return resources; - } - - - /** Returns the offset of the greatest index entry which is less than or equal to term.*/ - private final int getIndexOffset(Term term) { - int lo = 0; // binary search indexTerms[] - int hi = indexTerms.length - 1; - - while (hi >= lo) { - int mid = (lo + hi) >>> 1; - int delta = term.compareTo(indexTerms[mid]); - if (delta < 0) - hi = mid - 1; - else if (delta > 0) - lo = mid + 1; - else - return mid; - } - return hi; - } - - private final void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException { - enumerator.seek(indexPointers[indexOffset], - (indexOffset * totalIndexInterval) - 1, - indexTerms[indexOffset], indexInfos[indexOffset]); - } - - /** Returns the TermInfo for a Term in the set, or null. */ - TermInfo get(Term term) throws IOException { - return get(term, true); - } - - /** Returns the TermInfo for a Term in the set, or null. */ - private TermInfo get(Term term, boolean useCache) throws IOException { - if (size == 0) return null; - - ensureIndexIsRead(); - - TermInfo ti; - ThreadResources resources = getThreadResources(); - Cache cache = null; - - if (useCache) { - cache = resources.termInfoCache; - // check the cache first if the term was recently looked up - ti = (TermInfo) cache.get(term); - if (ti != null) { - return ti; - } - } - - // optimize sequential access: first try scanning cached enum w/o seeking - SegmentTermEnum enumerator = resources.termEnum; - if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && term.compareTo(enumerator.prev())> 0) - || term.compareTo(enumerator.term()) >= 0)) { - int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; - if (indexTerms.length == enumOffset // but before end of block - || term.compareTo(indexTerms[enumOffset]) < 0) { - // no need to seek - - int numScans = enumerator.scanTo(term); - if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) { - ti = enumerator.termInfo(); - if (cache != null && numScans > 1) { - // we only want to put this TermInfo into the cache if - // scanEnum skipped more than one dictionary entry. - // This prevents RangeQueries or WildcardQueries to - // wipe out the cache when they iterate over a large numbers - // of terms in order - cache.put(term, ti); - } - } else { - ti = null; - } - - return ti; - } - } - - // random-access: must seek - seekEnum(enumerator, getIndexOffset(term)); - enumerator.scanTo(term); - if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) { - ti = enumerator.termInfo(); - if (cache != null) { - cache.put(term, ti); - } - } else { - ti = null; - } - return ti; - } - - /** Returns the nth term in the set. */ - final Term get(int position) throws IOException { - if (size == 0) return null; - - SegmentTermEnum enumerator = getThreadResources().termEnum; - if (enumerator.term() != null && - position >= enumerator.position && - position < (enumerator.position + totalIndexInterval)) - return scanEnum(enumerator, position); // can avoid seek - - seekEnum(enumerator, position/totalIndexInterval); // must seek - return scanEnum(enumerator, position); - } - - private final Term scanEnum(SegmentTermEnum enumerator, int position) throws IOException { - while(enumerator.position < position) - if (!enumerator.next()) - return null; - - return enumerator.term(); - } - - private void ensureIndexIsRead() { - if (indexTerms == null) { - throw new IllegalStateException("terms index was not loaded when this reader was created"); - } - } - - /** Returns the position of a Term in the set or -1. */ - final long getPosition(Term term) throws IOException { - if (size == 0) return -1; - - ensureIndexIsRead(); - int indexOffset = getIndexOffset(term); - - SegmentTermEnum enumerator = getThreadResources().termEnum; - seekEnum(enumerator, indexOffset); - - while(term.compareTo(enumerator.term()) > 0 && enumerator.next()) {} - - if (term.compareTo(enumerator.term()) == 0) - return enumerator.position; - else - return -1; - } - - /** Returns an enumeration of all the Terms and TermInfos in the set. */ - public SegmentTermEnum terms() { - return (SegmentTermEnum)origEnum.clone(); - } - - /** Returns an enumeration of terms starting at or after the named term. */ - public SegmentTermEnum terms(Term term) throws IOException { - // don't use the cache in this call because we want to reposition the - // enumeration - get(term, false); - return (SegmentTermEnum)getThreadResources().termEnum.clone(); - } -} Index: src/java/org/apache/lucene/index/TermInfosWriter.java =================================================================== --- src/java/org/apache/lucene/index/TermInfosWriter.java (revision 822088) +++ src/java/org/apache/lucene/index/TermInfosWriter.java (working copy) @@ -1,228 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.io.IOException; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.UnicodeUtil; - -/** This stores a monotonically increasing set of pairs in a - Directory. A TermInfos can be written once, in order. */ - -final class TermInfosWriter { - /** The file format version, a negative number. */ - public static final int FORMAT = -3; - - // Changed strings to true utf8 with length-in-bytes not - // length-in-chars - public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; - - // NOTE: always change this if you switch to a new format! - public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - private FieldInfos fieldInfos; - private IndexOutput output; - private TermInfo lastTi = new TermInfo(); - private long size; - - // TODO: the default values for these two parameters should be settable from - // IndexWriter. However, once that's done, folks will start setting them to - // ridiculous values and complaining that things don't work well, as with - // mergeFactor. So, let's wait until a number of folks find that alternate - // values work better. Note that both of these values are stored in the - // segment, so that it's safe to change these w/o rebuilding all indexes. - - /** Expert: The fraction of terms in the "dictionary" which should be stored - * in RAM. Smaller values use more memory, but make searching slightly - * faster, while larger values use less memory and make searching slightly - * slower. Searching is typically not dominated by dictionary lookup, so - * tweaking this is rarely useful.*/ - int indexInterval = 128; - - /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, - * used to accelerate {@link TermDocs#skipTo(int)}. Larger values result in - * smaller indexes, greater acceleration, but fewer accelerable cases, while - * smaller values result in bigger indexes, less acceleration and more - * accelerable cases. More detailed experiments would be useful here. */ - int skipInterval = 16; - - /** Expert: The maximum number of skip levels. Smaller values result in - * slightly smaller indexes, but slower skipping in big posting lists. - */ - int maxSkipLevels = 10; - - private long lastIndexPointer; - private boolean isIndex; - private byte[] lastTermBytes = new byte[10]; - private int lastTermBytesLength = 0; - private int lastFieldNumber = -1; - - private TermInfosWriter other; - private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result(); - - TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval) - throws IOException { - initialize(directory, segment, fis, interval, false); - other = new TermInfosWriter(directory, segment, fis, interval, true); - other.other = this; - } - - private TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval, boolean isIndex) throws IOException { - initialize(directory, segment, fis, interval, isIndex); - } - - private void initialize(Directory directory, String segment, FieldInfos fis, - int interval, boolean isi) throws IOException { - indexInterval = interval; - fieldInfos = fis; - isIndex = isi; - output = directory.createOutput(segment + (isIndex ? ".tii" : ".tis")); - output.writeInt(FORMAT_CURRENT); // write format - output.writeLong(0); // leave space for size - output.writeInt(indexInterval); // write indexInterval - output.writeInt(skipInterval); // write skipInterval - output.writeInt(maxSkipLevels); // write maxSkipLevels - assert initUTF16Results(); - } - - void add(Term term, TermInfo ti) throws IOException { - UnicodeUtil.UTF16toUTF8(term.text, 0, term.text.length(), utf8Result); - add(fieldInfos.fieldNumber(term.field), utf8Result.result, utf8Result.length, ti); - } - - // Currently used only by assert statements - UnicodeUtil.UTF16Result utf16Result1; - UnicodeUtil.UTF16Result utf16Result2; - - // Currently used only by assert statements - private boolean initUTF16Results() { - utf16Result1 = new UnicodeUtil.UTF16Result(); - utf16Result2 = new UnicodeUtil.UTF16Result(); - return true; - } - - // Currently used only by assert statement - private int compareToLastTerm(int fieldNumber, byte[] termBytes, int termBytesLength) { - - if (lastFieldNumber != fieldNumber) { - final int cmp = fieldInfos.fieldName(lastFieldNumber).compareTo(fieldInfos.fieldName(fieldNumber)); - // If there is a field named "" (empty string) then we - // will get 0 on this comparison, yet, it's "OK". But - // it's not OK if two different field numbers map to - // the same name. - if (cmp != 0 || lastFieldNumber != -1) - return cmp; - } - - UnicodeUtil.UTF8toUTF16(lastTermBytes, 0, lastTermBytesLength, utf16Result1); - UnicodeUtil.UTF8toUTF16(termBytes, 0, termBytesLength, utf16Result2); - final int len; - if (utf16Result1.length < utf16Result2.length) - len = utf16Result1.length; - else - len = utf16Result2.length; - - for(int i=0;i, TermInfo> pair to the set. - Term must be lexicographically greater than all previous Terms added. - TermInfo pointers must be positive and greater than all previous.*/ - void add(int fieldNumber, byte[] termBytes, int termBytesLength, TermInfo ti) - throws IOException { - - assert compareToLastTerm(fieldNumber, termBytes, termBytesLength) < 0 || - (isIndex && termBytesLength == 0 && lastTermBytesLength == 0) : - "Terms are out of order: field=" + fieldInfos.fieldName(fieldNumber) + " (number " + fieldNumber + ")" + - " lastField=" + fieldInfos.fieldName(lastFieldNumber) + " (number " + lastFieldNumber + ")" + - " text=" + new String(termBytes, 0, termBytesLength, "UTF-8") + " lastText=" + new String(lastTermBytes, 0, lastTermBytesLength, "UTF-8"); - - assert ti.freqPointer >= lastTi.freqPointer: "freqPointer out of order (" + ti.freqPointer + " < " + lastTi.freqPointer + ")"; - assert ti.proxPointer >= lastTi.proxPointer: "proxPointer out of order (" + ti.proxPointer + " < " + lastTi.proxPointer + ")"; - - if (!isIndex && size % indexInterval == 0) - other.add(lastFieldNumber, lastTermBytes, lastTermBytesLength, lastTi); // add an index term - - writeTerm(fieldNumber, termBytes, termBytesLength); // write term - - output.writeVInt(ti.docFreq); // write doc freq - output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers - output.writeVLong(ti.proxPointer - lastTi.proxPointer); - - if (ti.docFreq >= skipInterval) { - output.writeVInt(ti.skipOffset); - } - - if (isIndex) { - output.writeVLong(other.output.getFilePointer() - lastIndexPointer); - lastIndexPointer = other.output.getFilePointer(); // write pointer - } - - lastFieldNumber = fieldNumber; - lastTi.set(ti); - size++; - } - - private void writeTerm(int fieldNumber, byte[] termBytes, int termBytesLength) - throws IOException { - - // TODO: UTF16toUTF8 could tell us this prefix - // Compute prefix in common with last term: - int start = 0; - final int limit = termBytesLength < lastTermBytesLength ? termBytesLength : lastTermBytesLength; - while(start < limit) { - if (termBytes[start] != lastTermBytes[start]) - break; - start++; - } - - final int length = termBytesLength - start; - output.writeVInt(start); // write shared prefix length - output.writeVInt(length); // write delta length - output.writeBytes(termBytes, start, length); // write delta bytes - output.writeVInt(fieldNumber); // write field num - if (lastTermBytes.length < termBytesLength) { - byte[] newArray = new byte[(int) (termBytesLength*1.5)]; - System.arraycopy(lastTermBytes, 0, newArray, 0, start); - lastTermBytes = newArray; - } - System.arraycopy(termBytes, start, lastTermBytes, start, length); - lastTermBytesLength = termBytesLength; - } - - /** Called to complete TermInfos creation. */ - void close() throws IOException { - output.seek(4); // write size after format - output.writeLong(size); - output.close(); - - if (!isIndex) - other.close(); - } - -} Index: src/java/org/apache/lucene/index/TermPositions.java =================================================================== --- src/java/org/apache/lucene/index/TermPositions.java (revision 822088) +++ src/java/org/apache/lucene/index/TermPositions.java (working copy) @@ -26,6 +26,7 @@ * positions of each occurrence of a term in a document. * * @see IndexReader#termPositions() + * @deprecated Use {@link PositionsEnum} instead */ public interface TermPositions Index: src/java/org/apache/lucene/index/TermRef.java =================================================================== --- src/java/org/apache/lucene/index/TermRef.java (revision 0) +++ src/java/org/apache/lucene/index/TermRef.java (revision 0) @@ -0,0 +1,150 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.ArrayUtil; +import java.io.UnsupportedEncodingException; + +/** Represents the UTF8 bytes[] for a term's text. This is + * used when reading with the flex API, to avoid having to + * materialize full char[]. */ +public class TermRef { + + public byte[] bytes; + public int offset; + public int length; + + public TermRef() { + } + + public TermRef(String text) { + copy(text); + } + + public void copy(String text) { + try { + bytes = text.getBytes("UTF-8"); + } catch (UnsupportedEncodingException uee) { + // should not happen: + throw new RuntimeException("unable to encode to UTF-8"); + } + offset = 0; + length = bytes.length; + } + + public int compareTerm(TermRef other) { + final int minLength; + if (length < other.length) { + minLength = length; + } else { + minLength = other.length; + } + int upto = offset; + int otherUpto = other.offset; + final byte[] otherBytes = other.bytes; + for(int i=0;i offset) { + sb.append(' '); + } + sb.append(""+bytes[i]); + } + sb.append(']'); + return sb.toString(); + } + + public void copy(TermRef other) { + bytes = ArrayUtil.grow(bytes, other.length); + System.arraycopy(other.bytes, other.offset, bytes, 0, other.length); + length = other.length; + offset = 0; + } + + public void grow(int newLength) { + bytes = ArrayUtil.grow(bytes, newLength); + } +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/index/TermRef.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/Terms.java =================================================================== --- src/java/org/apache/lucene/index/Terms.java (revision 0) +++ src/java/org/apache/lucene/index/Terms.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.util.Bits; + +/** + * NOTE: this API is experimental and will likely change + */ + +public abstract class Terms { + + // nocommit -- char[] or byte[] version? + /** Returns an iterator that will step through all terms */ + public abstract TermsEnum iterator() throws IOException; + + /** Returns the docFreq of the specified term text. */ + public int docFreq(TermRef text) throws IOException { + // nocommit -- make thread private cache so we share + // single enum + // NOTE: subclasses may have more efficient impl + final TermsEnum terms = iterator(); + TermsEnum.SeekResult result = terms.seek(text); + if (result.status == TermsEnum.SeekResult.Status.FOUND) { + return terms.docFreq(); + } else { + return 0; + } + } + + /** Get DocsEnum for the specified term. */ + public DocsEnum docs(Bits skipDocs, TermRef text) throws IOException { + // NOTE: subclasses may have more efficient impl + final TermsEnum terms = iterator(); + TermsEnum.SeekResult result = terms.seek(text); + if (result.status == TermsEnum.SeekResult.Status.FOUND) { + return terms.docs(skipDocs); + } else { + return null; + } + } + + public long getUniqueTermCount() throws IOException { + throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()"); + } +} Property changes on: src/java/org/apache/lucene/index/Terms.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/TermsEnum.java =================================================================== --- src/java/org/apache/lucene/index/TermsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/TermsEnum.java (revision 0) @@ -0,0 +1,66 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.Bits; + +/** + * NOTE: this API is experimental and will likely change + */ + +/** On obtaining a TermsEnum, you must first call next() */ +public abstract class TermsEnum extends AttributeSource { + + /** Represents returned result from {@link TermsEnum.seek}. + * If status is SEEK_FOUND or SEEK_END, the text is + * unused; else (status is SEEK_NOT_FOUND), the text is the + * term that was actually found. */ + public final static class SeekResult { + public static enum Status {END, FOUND, NOT_FOUND}; + public Status status; + public TermRef term; + } + + /** Seeks to the specified term. Returns SeekResult to + * indicate whether exact term was found, a different + * term was found, or EOF was hit. NOTE: the SeekResult + * instance may be reused across different calls to seek. */ + public abstract SeekResult seek(TermRef text) throws IOException; + + /** Increments the enumeration to the next element. + * Returns the resulting TermRef, or null if the end was + * hit. The TermRef may be re-used between calls to + * next. */ + public abstract TermRef next() throws IOException; + + // nocommit: add term()? + + /** Returns the docFreq of the current Term in the enumeration.*/ + public abstract int docFreq(); + + /** Get DocsEnum for the current term. The returned + * DocsEnum may shared state with this TermsEnum, so you + * should not call this TermsEnum's {@link #seek} or + * {@link #next} until you are done using the + * DocsEnum. */ + public abstract DocsEnum docs(Bits skipDocs) throws IOException; +} + Property changes on: src/java/org/apache/lucene/index/TermsEnum.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/TermsHashPerField.java =================================================================== --- src/java/org/apache/lucene/index/TermsHashPerField.java (revision 822088) +++ src/java/org/apache/lucene/index/TermsHashPerField.java (working copy) @@ -350,6 +350,8 @@ final char[] tokenText = termAtt.termBuffer();; final int tokenTextLen = termAtt.termLength(); + // System.out.println("thpf.add: field=" + fieldInfo.name + " text=" + new String(tokenText, 0, tokenTextLen) + " c0=" + ((int) tokenText[0]) ); + // Compute hashcode & replace any invalid UTF16 sequences int downto = tokenTextLen; int code = 0; Index: src/java/org/apache/lucene/index/codecs/Codec.java =================================================================== --- src/java/org/apache/lucene/index/codecs/Codec.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/Codec.java (revision 0) @@ -0,0 +1,94 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +public abstract class Codec { + + public static boolean DEBUG = false; + + private static final int CODEC_HEADER = 0x1af65; + + /** Unique name that's used to retrieve this codec when + * reading the index */ + public String name; + + /** Writes a new segment */ + public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException; + + /** Reads a segment */ + // nocommit -- add a "required capabilities" here; this + // way merging could say only "TERMS_LINEAR_SCAN" but + // searching would say "TERMS_RANDOM_ACCESS"? + public abstract FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException; + + /** Gathers files associated with this segment */ + public abstract void files(Directory dir, SegmentInfo segmentInfo, Collection files) throws IOException; + + /** Records all file extensions this codec uses */ + public abstract void getExtensions(Collection extensions); + + /** @return Actual version of the file */ + public static int checkHeader(IndexInput in, String codec, int version) throws IOException { + + // Safety to guard against reading a bogus string: + int header = in.readInt(); + if (header != CODEC_HEADER) { + throw new CorruptIndexException("codec header mismatch: " + header + " vs " + CODEC_HEADER); + } + + final String actualCodec = in.readString(); + if (!codec.equals(actualCodec)) { + throw new CorruptIndexException("codec mismatch: expected '" + codec + "' but got '" + actualCodec + "'"); + } + + int actualVersion = in.readInt(); + if (actualVersion > version) { + throw new CorruptIndexException("version '" + actualVersion + "' is too new (expected <= '" + version + "'"); + } + + return actualVersion; + } + + public static void writeHeader(IndexOutput out, String codec, int version) throws IOException { + final long start = out.getFilePointer(); + out.writeInt(CODEC_HEADER); + out.writeString(codec); + out.writeInt(version); + + // So we can easily compute headerSize (below) + if (out.getFilePointer()-start != codec.length() + 9) { + System.out.println(out.getFilePointer()-start + " vs " + (codec.length() + 8)); + throw new IllegalArgumentException("codec must be simple ASCII, less than 128 characters in length [got " + codec + "]"); + } + } + + public static int headerSize(String codec) { + return 9 + codec.length(); + } +} Property changes on: src/java/org/apache/lucene/index/codecs/Codec.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/Codecs.java =================================================================== --- src/java/org/apache/lucene/index/codecs/Codecs.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/Codecs.java (revision 0) @@ -0,0 +1,94 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.HashMap; +import java.util.Collection; +import java.util.HashSet; + +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; +import org.apache.lucene.index.codecs.sep.SepCodec; +import org.apache.lucene.index.codecs.intblock.IntBlockCodec; +import org.apache.lucene.index.codecs.pfordelta.PForDeltaCodec; +import org.apache.lucene.index.SegmentWriteState; + +/** Holds a set of codecs, keyed by name. You subclass + * this, instantiate it, and register your codecs, then + * pass this instance to IndexReader/IndexWriter (via + * package private APIs) to use different codecs when + * reading & writing segments. */ + +public abstract class Codecs { + + private final HashMap codecs = new HashMap(); + + private final Collection knownExtensions = new HashSet(); + + public void register(Codec codec) { + if (codec.name == null) { + throw new IllegalArgumentException("code.name is null"); + } + + if (!codecs.containsKey(codec.name)) { + codecs.put(codec.name, codec); + codec.getExtensions(knownExtensions); + } else if (codecs.get(codec.name) != codec) { + throw new IllegalArgumentException("codec '" + codec.name + "' is already registered as a different codec instance"); + } + } + + public Collection getAllExtensions() { + return knownExtensions; + } + + public Codec lookup(String name) { + final Codec codec = (Codec) codecs.get(name); + if (codec == null) + throw new IllegalArgumentException("required codec '" + name + "' not found"); + return codec; + } + + public abstract Codec getWriter(SegmentWriteState state); + + static private final Codecs defaultCodecs = new DefaultCodecs(); + + public static Codecs getDefault() { + return defaultCodecs; + } +} + +class DefaultCodecs extends Codecs { + DefaultCodecs() { + register(new StandardCodec()); + register(new IntBlockCodec()); + register(new PreFlexCodec()); + register(new PulsingCodec()); + register(new SepCodec()); + register(new PForDeltaCodec()); + } + + public Codec getWriter(SegmentWriteState state) { + return lookup("Standard"); + //return lookup("Pulsing"); + //return lookup("Sep"); + //return lookup("IntBlock"); + //return lookup("PForDelta"); + } +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/index/codecs/Codecs.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/DocsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/DocsConsumer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/DocsConsumer.java (revision 0) @@ -0,0 +1,55 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.FieldInfo; + +/** + * NOTE: this API is experimental and will likely change + */ + +public abstract class DocsConsumer { + + // nocommit + public String desc; + /* + public boolean setDesc(String desc) { + this.desc = desc; + return true; + } + */ + + public abstract void start(IndexOutput termsOut) throws IOException; + + public abstract void startTerm() throws IOException; + + /** Adds a new doc in this term. Return null if this + * consumer doesn't need to see the positions for this + * doc. */ + public abstract PositionsConsumer addDoc(int docID, int termDocFreq) throws IOException; + + /** Finishes the current term */ + public abstract void finishTerm(int numDocs, boolean isIndexTerm) throws IOException; + + public abstract void setField(FieldInfo fieldInfo); + + public abstract void close() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/codecs/DocsConsumer.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/DocsProducer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/DocsProducer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/DocsProducer.java (revision 0) @@ -0,0 +1,54 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.DocsEnum; + + +// nocommit -- this is tied to StandarTermsDictWriter; +// shouldn't it be named StandardDocsProducer? hmm, though, +// it's API is fairly generic in that any other terms dict +// codec could re-use it + +/** StandardTermsDictReader interacts with a single instance + * of this to manage creation of multiple docs enum + * instances. It provides an IndexInput (termsIn) where + * this class may read any previously stored data that it + * had written in its corresponding DocsConsumer. */ +public abstract class DocsProducer { + + public abstract class Reader { + public abstract void readTerm(int docFreq, boolean isIndexTerm) throws IOException; + + /** Returns a docs enum for the last term read */ + public abstract DocsEnum docs(Bits deletedDocs) throws IOException; + } + + public abstract void start(IndexInput termsIn) throws IOException; + + /** Returns a new private reader for stepping through + * terms, getting DocsEnum. */ + public abstract Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException; + + public abstract void close() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/codecs/DocsProducer.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/FieldsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (revision 0) @@ -0,0 +1,38 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.FieldInfo; + +import java.io.IOException; + +/** Abstract API that consumes terms, doc, freq, prox and + * payloads postings. Concrete implementations of this + * actually do "something" with the postings (write it into + * the index in a specific format). + * + * NOTE: this API is experimental and will likely change + */ +public abstract class FieldsConsumer { + + /** Add a new field */ + public abstract TermsConsumer addField(FieldInfo field) throws IOException; + + /** Called when we are done adding everything. */ + public abstract void close() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/codecs/FieldsConsumer.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/FieldsProducer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/FieldsProducer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/FieldsProducer.java (revision 0) @@ -0,0 +1,34 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.Fields; + +import java.io.IOException; + +/** Abstract API that consumes terms, doc, freq, prox and + * payloads postings. Concrete implementations of this + * actually do "something" with the postings (write it into + * the index in a specific format). + * + * NOTE: this API is experimental and will likely change + */ +public abstract class FieldsProducer extends Fields { + public abstract void close() throws IOException; + public abstract void loadTermsIndex() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/codecs/FieldsProducer.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/MultiLevelSkipListReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/MultiLevelSkipListReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/MultiLevelSkipListReader.java (revision 0) @@ -0,0 +1,279 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.BufferedIndexInput; +import org.apache.lucene.store.IndexInput; + +/** + * This abstract class reads skip lists with multiple levels. + * + * See {@link MultiLevelSkipListWriter} for the information about the encoding + * of the multi level skip lists. + * + * Subclasses must implement the abstract method {@link #readSkipData(int, IndexInput)} + * which defines the actual format of the skip data. + */ + +// nocommit -- made public +public abstract class MultiLevelSkipListReader { + // the maximum number of skip levels possible for this index + protected int maxNumberOfSkipLevels; + + // number of levels in this skip list + private int numberOfSkipLevels; + + // Expert: defines the number of top skip levels to buffer in memory. + // Reducing this number results in less memory usage, but possibly + // slower performance due to more random I/Os. + // Please notice that the space each level occupies is limited by + // the skipInterval. The top level can not contain more than + // skipLevel entries, the second top level can not contain more + // than skipLevel^2 entries and so forth. + private int numberOfLevelsToBuffer = 1; + + private int docCount; + private boolean haveSkipped; + + private IndexInput[] skipStream; // skipStream for each level + private long skipPointer[]; // the start pointer of each skip level + private int skipInterval[]; // skipInterval of each level + private int[] numSkipped; // number of docs skipped per level + + private int[] skipDoc; // doc id of current skip entry per level + private int lastDoc; // doc id of last read skip entry with docId <= target + private long[] childPointer; // child pointer of current skip entry per level + private long lastChildPointer; // childPointer of last read skip entry with docId <= target + + private boolean inputIsBuffered; + + public MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) { + this.skipStream = new IndexInput[maxSkipLevels]; + this.skipPointer = new long[maxSkipLevels]; + this.childPointer = new long[maxSkipLevels]; + this.numSkipped = new int[maxSkipLevels]; + this.maxNumberOfSkipLevels = maxSkipLevels; + this.skipInterval = new int[maxSkipLevels]; + this.skipStream [0]= skipStream; + this.inputIsBuffered = (skipStream instanceof BufferedIndexInput); + this.skipInterval[0] = skipInterval; + for (int i = 1; i < maxSkipLevels; i++) { + // cache skip intervals + this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval; + } + skipDoc = new int[maxSkipLevels]; + } + + + /** Returns the id of the doc to which the last call of {@link #skipTo(int)} + * has skipped. */ + // nocommit made public + public int getDoc() { + return lastDoc; + } + + + /** Skips entries to the first beyond the current whose document number is + * greater than or equal to target. Returns the current doc count. + */ + // nocommit made public + public int skipTo(int target) throws IOException { + if (!haveSkipped) { + // first time, load skip levels + loadSkipLevels(); + haveSkipped = true; + } + + // walk up the levels until highest level is found that has a skip + // for this target + int level = 0; + while (level < numberOfSkipLevels - 1 && target > skipDoc[level + 1]) { + level++; + } + + while (level >= 0) { + if (target > skipDoc[level]) { + if (!loadNextSkip(level)) { + continue; + } + } else { + // no more skips on this level, go down one level + if (level > 0 && lastChildPointer > skipStream[level - 1].getFilePointer()) { + seekChild(level - 1); + } + level--; + } + } + + return numSkipped[0] - skipInterval[0] - 1; + } + + private boolean loadNextSkip(int level) throws IOException { + // we have to skip, the target document is greater than the current + // skip list entry + setLastSkipData(level); + + numSkipped[level] += skipInterval[level]; + + if (numSkipped[level] > docCount) { + // this skip list is exhausted + skipDoc[level] = Integer.MAX_VALUE; + if (numberOfSkipLevels > level) numberOfSkipLevels = level; + return false; + } + + // read next skip entry + skipDoc[level] += readSkipData(level, skipStream[level]); + + if (level != 0) { + // read the child pointer if we are not on the leaf level + childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1]; + } + + return true; + + } + + /** Seeks the skip entry on the given level */ + protected void seekChild(int level) throws IOException { + skipStream[level].seek(lastChildPointer); + numSkipped[level] = numSkipped[level + 1] - skipInterval[level + 1]; + skipDoc[level] = lastDoc; + if (level > 0) { + childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1]; + } + } + + // nocommit -- made public + public void close() throws IOException { + for (int i = 1; i < skipStream.length; i++) { + if (skipStream[i] != null) { + skipStream[i].close(); + } + } + } + + /** initializes the reader */ + // nocommit -- made public + public void init(long skipPointer, int df) { + this.skipPointer[0] = skipPointer; + this.docCount = df; + Arrays.fill(skipDoc, 0); + Arrays.fill(numSkipped, 0); + Arrays.fill(childPointer, 0); + + haveSkipped = false; + for (int i = 1; i < numberOfSkipLevels; i++) { + skipStream[i] = null; + } + } + + /** Loads the skip levels */ + private void loadSkipLevels() throws IOException { + numberOfSkipLevels = docCount == 0 ? 0 : (int) Math.floor(Math.log(docCount) / Math.log(skipInterval[0])); + if (numberOfSkipLevels > maxNumberOfSkipLevels) { + numberOfSkipLevels = maxNumberOfSkipLevels; + } + + skipStream[0].seek(skipPointer[0]); + + int toBuffer = numberOfLevelsToBuffer; + + for (int i = numberOfSkipLevels - 1; i > 0; i--) { + // the length of the current level + long length = skipStream[0].readVLong(); + + // the start pointer of the current level + skipPointer[i] = skipStream[0].getFilePointer(); + if (toBuffer > 0) { + // buffer this level + skipStream[i] = new SkipBuffer(skipStream[0], (int) length); + toBuffer--; + } else { + // clone this stream, it is already at the start of the current level + skipStream[i] = (IndexInput) skipStream[0].clone(); + if (inputIsBuffered && length < BufferedIndexInput.BUFFER_SIZE) { + ((BufferedIndexInput) skipStream[i]).setBufferSize((int) length); + } + + // move base stream beyond the current level + skipStream[0].seek(skipStream[0].getFilePointer() + length); + } + } + + // use base stream for the lowest level + skipPointer[0] = skipStream[0].getFilePointer(); + } + + /** + * Subclasses must implement the actual skip data encoding in this method. + * + * @param level the level skip data shall be read from + * @param skipStream the skip stream to read from + */ + protected abstract int readSkipData(int level, IndexInput skipStream) throws IOException; + + /** Copies the values of the last read skip entry on this level */ + protected void setLastSkipData(int level) { + lastDoc = skipDoc[level]; + lastChildPointer = childPointer[level]; + } + + + /** used to buffer the top skip levels */ + private final static class SkipBuffer extends IndexInput { + private byte[] data; + private long pointer; + private int pos; + + SkipBuffer(IndexInput input, int length) throws IOException { + data = new byte[length]; + pointer = input.getFilePointer(); + input.readBytes(data, 0, length); + } + + public void close() throws IOException { + data = null; + } + + public long getFilePointer() { + return pointer + pos; + } + + public long length() { + return data.length; + } + + public byte readByte() throws IOException { + return data[pos++]; + } + + public void readBytes(byte[] b, int offset, int len) throws IOException { + System.arraycopy(data, pos, b, offset, len); + pos += len; + } + + public void seek(long pos) throws IOException { + this.pos = (int) (pos - pointer); + } + + } +} Index: src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java (revision 0) @@ -0,0 +1,156 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; + +/** + * This abstract class writes skip lists with multiple levels. + * + * Example for skipInterval = 3: + * c (skip level 2) + * c c c (skip level 1) + * x x x x x x x x x x (skip level 0) + * d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d (posting list) + * 3 6 9 12 15 18 21 24 27 30 (df) + * + * d - document + * x - skip data + * c - skip data with child pointer + * + * Skip level i contains every skipInterval-th entry from skip level i-1. + * Therefore the number of entries on level i is: floor(df / ((skipInterval ^ (i + 1))). + * + * Each skip entry on a level i>0 contains a pointer to the corresponding skip entry in list i-1. + * This guarantees a logarithmic amount of skips to find the target document. + * + * While this class takes care of writing the different skip levels, + * subclasses must define the actual format of the skip data. + * + */ + +// nocommit -- made public +public abstract class MultiLevelSkipListWriter { + // number of levels in this skip list + protected int numberOfSkipLevels; + + // the skip interval in the list with level = 0 + private int skipInterval; + + // for every skip level a different buffer is used + private RAMOutputStream[] skipBuffer; + + protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) { + this.skipInterval = skipInterval; + + // calculate the maximum number of skip levels for this document frequency + numberOfSkipLevels = df == 0 ? 0 : (int) Math.floor(Math.log(df) / Math.log(skipInterval)); + + // make sure it does not exceed maxSkipLevels + if (numberOfSkipLevels > maxSkipLevels) { + numberOfSkipLevels = maxSkipLevels; + } + } + + protected void init() { + skipBuffer = new RAMOutputStream[numberOfSkipLevels]; + for (int i = 0; i < numberOfSkipLevels; i++) { + skipBuffer[i] = new RAMOutputStream(); + } + } + + protected void resetSkip() { + // creates new buffers or empties the existing ones + if (skipBuffer == null) { + init(); + } else { + for (int i = 0; i < skipBuffer.length; i++) { + skipBuffer[i].reset(); + } + } + } + + /** + * Subclasses must implement the actual skip data encoding in this method. + * + * @param level the level skip data shall be writing for + * @param skipBuffer the skip buffer to write to + */ + protected abstract void writeSkipData(int level, IndexOutput skipBuffer) throws IOException; + + /** + * Writes the current skip data to the buffers. The current document frequency determines + * the max level is skip data is to be written to. + * + * @param df the current document frequency + * @throws IOException + */ + // nocommit -- made public + public void bufferSkip(int df) throws IOException { + int numLevels; + + // determine max level + for (numLevels = 0; (df % skipInterval) == 0 && numLevels < numberOfSkipLevels; df /= skipInterval) { + numLevels++; + } + + long childPointer = 0; + + for (int level = 0; level < numLevels; level++) { + writeSkipData(level, skipBuffer[level]); + + long newChildPointer = skipBuffer[level].getFilePointer(); + + if (level != 0) { + // store child pointers for all levels except the lowest + skipBuffer[level].writeVLong(childPointer); + } + + //remember the childPointer for the next level + childPointer = newChildPointer; + } + } + + /** + * Writes the buffered skip lists to the given output. + * + * @param output the IndexOutput the skip lists shall be written to + * @return the pointer the skip list starts + */ + // nocommit -- made public + public long writeSkip(IndexOutput output) throws IOException { + long skipPointer = output.getFilePointer(); + //System.out.println("skipper.writeSkip fp=" + skipPointer); + if (skipBuffer == null || skipBuffer.length == 0) return skipPointer; + + for (int level = numberOfSkipLevels - 1; level > 0; level--) { + long length = skipBuffer[level].getFilePointer(); + if (length > 0) { + output.writeVLong(length); + skipBuffer[level].writeTo(output); + } + } + skipBuffer[0].writeTo(output); + + return skipPointer; + } + +} Index: src/java/org/apache/lucene/index/codecs/PositionsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/PositionsConsumer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/PositionsConsumer.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; + +public abstract class PositionsConsumer { + + public abstract void start(IndexOutput termsOut) throws IOException; + + public abstract void startTerm() throws IOException; + + /** Add a new position & payload. If payloadLength > 0 + * you must read those bytes from the IndexInput. NOTE: + * you must fully consume the byte[] payload, since + * caller is free to reuse it on subsequent calls. */ + public abstract void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException; + + /** Called when we are done adding positions & payloads + * for each doc */ + public abstract void finishDoc() throws IOException; + + public abstract void finishTerm(boolean isIndexTerm) throws IOException; + + public abstract void close() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/codecs/PositionsConsumer.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/PositionsProducer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/PositionsProducer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/PositionsProducer.java (revision 0) @@ -0,0 +1,40 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.PositionsEnum; + +public abstract class PositionsProducer { + + public abstract class Reader { + public abstract void readTerm(int docFreq, boolean isIndexTerm) throws IOException; + + /** Returns a pos enum for the last term read */ + public abstract PositionsEnum positions() throws IOException; + } + + public abstract void start(IndexInput termsIn) throws IOException; + + public abstract Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException; + + public abstract void close() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/codecs/PositionsProducer.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/TermsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/TermsConsumer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/TermsConsumer.java (revision 0) @@ -0,0 +1,38 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** + * NOTE: this API is experimental and will likely change + */ + +public abstract class TermsConsumer { + + // nocommit -- CharSequence? + /** Starts a new term in this field; term ends with U+FFFF + * char */ + public abstract DocsConsumer startTerm(char[] text, int start) throws IOException; + + /** Finishes the current term */ + public abstract void finishTerm(char[] text, int start, int numDocs) throws IOException; + + /** Called when we are done adding terms to this field */ + public abstract void finish() throws IOException; +} Property changes on: src/java/org/apache/lucene/index/codecs/TermsConsumer.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java =================================================================== --- src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java (revision 0) @@ -0,0 +1,173 @@ +package org.apache.lucene.index.codecs.intblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Naive int block API that writes vInts. This is + * expected to give poor performance; it's really only for + * testing the pluggability. One should typically use pfor instead. */ + +import java.io.IOException; + +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.store.IndexInput; + +/** Abstract base class that reads fixed-size blocks of ints + * from an IndexInput. While this is a simple approach, a + * more performant approach would directly create an impl + * of IntIndexInput inside Directory. Wrapping a generic + * IndexInput will likely cost performance. */ +public abstract class FixedIntBlockIndexInput extends IntIndexInput { + + private IndexInput in; + protected int blockSize; + + protected void init(IndexInput in) throws IOException { + this.in = in; + blockSize = in.readVInt(); + } + + public Reader reader() throws IOException { + int[] buffer = new int[blockSize]; + IndexInput clone = (IndexInput) in.clone(); + // nocommit -- awkward + return new Reader(clone, buffer, getBlockReader(clone, buffer)); + } + + public void close() throws IOException { + in.close(); + } + + public Index index() { + return new Index(); + } + + protected abstract BlockReader getBlockReader(IndexInput in, int[] buffer) throws IOException; + + public interface BlockReader { + public void readBlock() throws IOException; + } + + private static class Reader extends IntIndexInput.Reader { + private final IndexInput in; + + protected final int[] pending; + int upto; + + private boolean seekPending; + private long pendingFP; + private int pendingUpto; + private long lastBlockFP; + private final BlockReader blockReader; + private final int blockSize; + + private final BulkReadResult result = new BulkReadResult(); + + public Reader(IndexInput in, int[] pending, BlockReader blockReader) { + this.in = in; + this.pending = pending; + this.blockSize = pending.length; + result.buffer = pending; + this.blockReader = blockReader; + } + + void seek(long fp, int upto) { + pendingFP = fp; + pendingUpto = upto; + seekPending = true; + } + + private void maybeSeek() throws IOException { + if (seekPending) { + if (pendingFP != lastBlockFP) { + // need new block + in.seek(pendingFP); + lastBlockFP = pendingFP; + blockReader.readBlock(); + } + upto = pendingUpto; + seekPending = false; + } + } + + public int next() throws IOException { + maybeSeek(); + if (upto == blockSize) { + lastBlockFP = in.getFilePointer(); + blockReader.readBlock(); + upto = 0; + } + + return pending[upto++]; + } + + public BulkReadResult read(int[] buffer, int count) throws IOException { + maybeSeek(); + if (upto == blockSize) { + blockReader.readBlock(); + upto = 0; + } + result.offset = upto; + if (upto + count < blockSize) { + result.len = count; + upto += count; + } else { + result.len = blockSize - upto; + upto = blockSize; + } + + return result; + } + + public String descFilePointer() { + return in.getFilePointer() + ":" + upto; + } + } + + private class Index extends IntIndexInput.Index { + private long fp; + private int upto; + + public void read(IndexInput indexIn, boolean absolute) throws IOException { + if (absolute) { + fp = indexIn.readVLong(); + upto = indexIn.readVInt(); + } else { + final long delta = indexIn.readVLong(); + if (delta == 0) { + // same block + upto += indexIn.readVInt(); + } else { + // new block + fp += delta; + upto = indexIn.readVInt(); + } + } + assert upto < blockSize; + } + + public void seek(IntIndexInput.Reader other) throws IOException { + ((Reader) other).seek(fp, upto); + } + + public void set(IntIndexInput.Index other) { + Index idx = (Index) other; + fp = idx.fp; + upto = idx.upto; + } + } +} \ No newline at end of file Index: src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java =================================================================== --- src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java (revision 0) @@ -0,0 +1,109 @@ +package org.apache.lucene.index.codecs.intblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Naive int block API that writes vInts. This is + * expected to give poor performance; it's really only for + * testing the pluggability. One should typically use pfor instead. */ + +import java.io.IOException; + +import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.store.IndexOutput; + +public abstract class FixedIntBlockIndexOutput extends IntIndexOutput { + + private IndexOutput out; + private int blockSize; + private int[] pending; + private int upto; + private long lastSavedFilePointer; + private int lastSavedUpto; + + protected void init(IndexOutput out, int fixedBlockSize) throws IOException { + blockSize = fixedBlockSize; + out.writeVInt(blockSize); + this.out = out; + pending = new int[blockSize]; + } + + protected abstract void flushBlock(int[] buffer, IndexOutput out) throws IOException; + + public Index index() throws IOException { + return new Index(); + } + + public String descFilePointer() { + return out.getFilePointer() + ":" + upto; + } + + private class Index extends IntIndexOutput.Index { + long fp; + int upto; + long lastFP; + int lastUpto; + + public void mark() throws IOException { + fp = out.getFilePointer(); + upto = FixedIntBlockIndexOutput.this.upto; + } + + public void set(IntIndexOutput.Index other) throws IOException { + Index idx = (Index) other; + lastFP = fp = idx.fp; + lastUpto = upto = idx.upto; + } + + public void write(IndexOutput indexOut, boolean absolute) throws IOException { + if (absolute) { + indexOut.writeVLong(fp); + indexOut.writeVInt(upto); + } else if (fp == lastFP) { + // same block + indexOut.writeVLong(0); + assert upto >= lastUpto; + indexOut.writeVLong(upto - lastUpto); + } else { + // new block + indexOut.writeVLong(fp - lastFP); + indexOut.writeVLong(upto); + } + lastUpto = upto; + lastFP = fp; + } + } + + public void write(int v) throws IOException { + pending[upto++] = v; + if (upto == blockSize) { + flushBlock(pending, out); + upto = 0; + } + } + + public void close() throws IOException { + // NOTE: entries in the block after current upto are + // invalid + // nocommit -- zero fill? + try { + flushBlock(pending, out); + } finally { + out.close(); + } + } +} Index: src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java =================================================================== --- src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (revision 0) @@ -0,0 +1,131 @@ +package org.apache.lucene.index.codecs.intblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocsConsumer; +import org.apache.lucene.index.codecs.DocsProducer; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.sep.SepCodec; +import org.apache.lucene.index.codecs.sep.SepDocsReader; +import org.apache.lucene.index.codecs.sep.SepDocsWriter; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsDictReader; +import org.apache.lucene.index.codecs.standard.StandardTermsDictWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexWriter; +import org.apache.lucene.store.Directory; + +public class IntBlockCodec extends Codec { + + public IntBlockCodec() { + name = "IntBlock"; + } + + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + DocsConsumer docsWriter = new SepDocsWriter(state, new SimpleIntBlockFactory(1024)); + + boolean success = false; + StandardTermsIndexWriter indexWriter; + try { + indexWriter = new SimpleStandardTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, docsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + docsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + /* + final static String DOC_EXTENSION = "doc"; + final static String SKIP_EXTENSION = "skp"; + final static String FREQ_EXTENSION = "frq"; + final static String PROX_EXTENSION = "prx"; + final static String PAYLOAD_EXTENSION = "pyl"; + */ + + public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException { + DocsProducer docsReader = new SepDocsReader(dir, si, readBufferSize, new SimpleIntBlockFactory(1024)); + + StandardTermsIndexReader indexReader; + boolean success = false; + try { + indexReader = new SimpleStandardTermsIndexReader(dir, + fieldInfos, + si.name, + indexDivisor); + success = true; + } finally { + if (!success) { + docsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new StandardTermsDictReader(indexReader, + dir, fieldInfos, si.name, + docsReader, + readBufferSize); + success = true; + return ret; + } finally { + if (!success) { + try { + docsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + public void files(Directory dir, SegmentInfo segmentInfo, Collection files) { + SepDocsReader.files(segmentInfo, files); + StandardTermsDictReader.files(segmentInfo, files); + SimpleStandardTermsIndexReader.files(segmentInfo, files); + } + + public void getExtensions(Collection extensions) { + SepCodec.getSepExtensions(extensions); + } +} Index: src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java =================================================================== --- src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java (revision 0) @@ -0,0 +1,38 @@ +package org.apache.lucene.index.codecs.intblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; + +import java.io.IOException; + +public class SimpleIntBlockFactory extends IntStreamFactory { + private final int blockSize; + public SimpleIntBlockFactory(int blockSize) { + this.blockSize = blockSize; + } + public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { + return new SimpleIntBlockIndexInput(dir, fileName, readBufferSize); + } + public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { + return new SimpleIntBlockIndexOutput(dir, fileName, blockSize); + } +} Index: src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java =================================================================== --- src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.index.codecs.intblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Naive int block API that writes vInts. This is + * expected to give poor performance; it's really only for + * testing the pluggability. One should typically use pfor instead. */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; + +import java.io.IOException; + +/** Don't use this class!! It naively encodes ints one vInt + * at a time. Use it only for testing. */ +public class SimpleIntBlockIndexInput extends FixedIntBlockIndexInput { + + public SimpleIntBlockIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException { + IndexInput in = dir.openInput(fileName, readBufferSize); + Codec.checkHeader(in, SimpleIntBlockIndexOutput.CODEC, SimpleIntBlockIndexOutput.VERSION_START); + init(in); + } + + private static class BlockReader implements FixedIntBlockIndexInput.BlockReader { + + private final IndexInput in; + private final int[] buffer; + + public BlockReader(IndexInput in, int[] buffer) { + this.in = in; + this.buffer = buffer; + } + + public void readBlock() throws IOException { + // silly impl + for(int i=0;i FieldInfo */ + + PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor) + throws IOException { + tis = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor); + this.fieldInfos = fieldInfos; + + // make sure that all index files have been read or are kept open + // so that if an index update removes them we'll still have them + freqStream = dir.openInput(info.name + ".frq", readBufferSize); + boolean anyProx = false; + final int numFields = fieldInfos.size(); + for(int i=0;i 0: "indexInterval=" + indexInterval + " is negative; must be > 0"; assert skipInterval > 0: "skipInterval=" + skipInterval + " is negative; must be > 0"; } - if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { + if (format > FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { termBuffer.setPreUTF8Strings(); scanBuffer.setPreUTF8Strings(); prevBuffer.setPreUTF8Strings(); Index: src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java =================================================================== --- src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.index; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -18,12 +18,17 @@ */ import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.FieldInfos; import java.io.IOException; -final class SegmentTermPositions +public final class SegmentTermPositions extends SegmentTermDocs implements TermPositions { private IndexInput proxStream; + private IndexInput proxStreamOrig; private int proxCount; private int position; @@ -37,11 +42,19 @@ // for a lazy skip private long lazySkipPointer = -1; private int lazySkipProxCount = 0; - + + /* SegmentTermPositions(SegmentReader p) { super(p); this.proxStream = null; // the proxStream will be cloned lazily when nextPosition() is called for the first time } + */ + + // nocommit -- public + public SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, Bits skipDocs, TermInfosReader tis, FieldInfos fieldInfos) { + super(freqStream, skipDocs, tis, fieldInfos); + this.proxStreamOrig = proxStream; // the proxStream will be cloned lazily when nextPosition() is called for the first time + } final void seek(TermInfo ti, Term term) throws IOException { super.seek(ti, term); @@ -146,7 +159,7 @@ private void lazySkip() throws IOException { if (proxStream == null) { // clone lazily - proxStream = (IndexInput) parent.core.proxStream.clone(); + proxStream = (IndexInput)proxStreamOrig.clone(); } // we might have to skip the current payload Index: src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java =================================================================== --- src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.index; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -20,6 +20,8 @@ import java.io.IOException; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.FieldInfos; final class TermBuffer implements Cloneable { Index: src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java =================================================================== --- src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.index; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,7 +17,10 @@ * limitations under the License. */ -/** A TermInfo is the record of information stored for a term.*/ +/** A TermInfo is the record of information stored for a + * term + * @deprecated This class is no longer used in flexible + * indexing. */ final class TermInfo { /** The number of documents which contain the term. */ Index: src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.index; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -19,16 +19,21 @@ import java.io.IOException; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.CloseableThreadLocal; import org.apache.lucene.util.cache.Cache; import org.apache.lucene.util.cache.SimpleLRUCache; -import org.apache.lucene.util.CloseableThreadLocal; /** This stores a monotonically increasing set of pairs in a * Directory. Pairs are accessed either by Term or by ordinal position the - * set. */ - -final class TermInfosReader { + * set + * @deprecated This class has been replaced by + * FormatPostingsTermsDictReader, except for reading old segments. */ +// nocommit -- public +public final class TermInfosReader { private final Directory directory; private final String segment; private final FieldInfos fieldInfos; @@ -68,7 +73,7 @@ segment = seg; fieldInfos = fis; - origEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_EXTENSION, + origEnum = new SegmentTermEnum(directory.openInput(segment + "." + PreFlexCodec.TERMS_EXTENSION, readBufferSize), fieldInfos, false); size = origEnum.size; @@ -76,7 +81,7 @@ if (indexDivisor != -1) { // Load terms index totalIndexInterval = origEnum.indexInterval * indexDivisor; - final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, + final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + PreFlexCodec.TERMS_INDEX_EXTENSION, readBufferSize), fieldInfos, true); try { @@ -197,7 +202,10 @@ return ti; } } - + + // nocommit -- make sure these optimizations survive + // into flex + // optimize sequential access: first try scanning cached enum w/o seeking SegmentTermEnum enumerator = resources.termEnum; if (enumerator.term() != null // term is at or past current @@ -289,7 +297,7 @@ /** Returns an enumeration of all the Terms and TermInfos in the set. */ public SegmentTermEnum terms() { - return (SegmentTermEnum)origEnum.clone(); + return (SegmentTermEnum) origEnum.clone(); } /** Returns an enumeration of terms starting at or after the named term. */ Index: src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java =================================================================== --- src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (revision 0) @@ -0,0 +1,146 @@ +package org.apache.lucene.index.codecs.pulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocsConsumer; +import org.apache.lucene.index.codecs.DocsProducer; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexWriter; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.standard.StandardDocsReader; +import org.apache.lucene.index.codecs.standard.StandardDocsWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsDictReader; +import org.apache.lucene.index.codecs.standard.StandardTermsDictWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexWriter; +import org.apache.lucene.store.Directory; + +/** This codec "inlines" the postings for terms that have + * low docFreq. It wraps another codec, which is used for + * writing the non-inlined terms. + * + * Currently in only inlines docFreq=1 terms, and + * otherwise uses the normal "standard" codec. */ + +public class PulsingCodec extends Codec { + + public PulsingCodec() { + name = "Pulsing"; + } + + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + // We wrap StandardDocsWriter, but any DocsConsumer + // will work: + DocsConsumer docsWriter = new StandardDocsWriter(state); + + // Terms that have <= freqCutoff number of docs are + // "pulsed" (inlined): + final int freqCutoff = 1; + DocsConsumer pulsingWriter = new PulsingDocsWriter(state, freqCutoff, docsWriter); + + // Terms dict index + StandardTermsIndexWriter indexWriter; + boolean success = false; + try { + indexWriter = new SimpleStandardTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + pulsingWriter.close(); + } + } + + // Terms dict + success = false; + try { + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, pulsingWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + pulsingWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException { + + // We wrap StandardDocsReader, but any DocsProducer + // will work: + DocsProducer docs = new StandardDocsReader(dir, si, readBufferSize); + DocsProducer docsReader = new PulsingDocsReader(dir, si, readBufferSize, docs); + + // Terms dict index reader + StandardTermsIndexReader indexReader; + + boolean success = false; + try { + indexReader = new SimpleStandardTermsIndexReader(dir, + fieldInfos, + si.name, + indexDivisor); + success = true; + } finally { + if (!success) { + docs.close(); + } + } + + // Terms dict reader + success = false; + try { + FieldsProducer ret = new StandardTermsDictReader(indexReader, + dir, fieldInfos, si.name, + docsReader, + readBufferSize); + success = true; + return ret; + } finally { + if (!success) { + try { + docs.close(); + } finally { + indexReader.close(); + } + } + } + } + + public void files(Directory dir, SegmentInfo segmentInfo, Collection files) { + StandardDocsReader.files(segmentInfo, files); + StandardTermsDictReader.files(segmentInfo, files); + SimpleStandardTermsIndexReader.files(segmentInfo, files); + } + + public void getExtensions(Collection extensions) { + StandardCodec.getStandardExtensions(extensions); + } +} Property changes on: src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java (revision 0) @@ -0,0 +1,303 @@ +package org.apache.lucene.index.codecs.pulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.PositionsEnum; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocsProducer; +import org.apache.lucene.index.codecs.pulsing.PulsingDocsWriter.Document; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; + +/** Concrete class that reads the current doc/freq/skip + * postings format */ + +// nocommit -- should we switch "hasProx" higher up? and +// create two separate docs readers, one that also reads +// prox and one that doesn't? + +class PulsingDocsReader extends DocsProducer { + + // Fallback reader for non-pulsed terms: + final DocsProducer wrappedDocsReader; + IndexInput termsIn; + int maxPulsingDocFreq; + + PulsingDocsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, DocsProducer wrappedDocsReader) throws IOException { + this.wrappedDocsReader = wrappedDocsReader; + } + + public void start(IndexInput termsIn) throws IOException { + this.termsIn = termsIn; + Codec.checkHeader(termsIn, PulsingDocsWriter.CODEC, PulsingDocsWriter.VERSION_START); + maxPulsingDocFreq = termsIn.readVInt(); + wrappedDocsReader.start(termsIn); + } + + public Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException { + return new PulsingReader(fieldInfo, termsIn, wrappedDocsReader.reader(fieldInfo, termsIn)); + } + + class PulsingReader extends Reader { + + final IndexInput termsIn; + final FieldInfo fieldInfo; + final boolean omitTF; + final boolean storePayloads; + int docFreq; + + // Holds pulsed docs + final Document[] docs; + + private boolean pendingIndexTerm; + private final Reader wrappedReader; + + PulsingReader(FieldInfo fieldInfo, IndexInput termsIn, Reader wrappedReader) { + this.termsIn = termsIn; // not cloned + this.fieldInfo = fieldInfo; + this.wrappedReader = wrappedReader; + omitTF = fieldInfo.omitTermFreqAndPositions; + storePayloads = fieldInfo.storePayloads; + docs = new Document[maxPulsingDocFreq]; + for(int i=0;i maxPulsingDocFreq docs + + static class Position { + byte[] payload; + int pos; + int payloadLength; + } + + // nocommit -- lazy init this? ie, if every single term + // was pulsed then we never need to use this fallback? + // Fallback writer for non-pulsed terms: + final DocsConsumer wrappedDocsWriter; + + /** If docFreq <= maxPulsingDocFreq, its postings are + * inlined into terms dict */ + PulsingDocsWriter(SegmentWriteState state, int maxPulsingDocFreq, DocsConsumer wrappedDocsWriter) throws IOException { + super(); + + pendingDocs = new Document[maxPulsingDocFreq]; + for(int i=0;i 0) { + if (pos.payload == null || payloadLength > pos.payload.length) { + pos.payload = new byte[ArrayUtil.getNextSize(payloadLength)]; + } + System.arraycopy(payload, payloadOffset, pos.payload, 0, payloadLength); + pos.payloadLength = payloadLength; + } else + pos.payloadLength = 0; + } + public void finishDoc() { + assert currentDoc.numPositions == currentDoc.termDocFreq; + } + public void finishTerm(boolean isIndexTerm) {} + public void close() {} + } + + final PositionsWriter posWriter = new PositionsWriter(); + + public PositionsConsumer addDoc(int docID, int termDocFreq) throws IOException { + + assert docID >= 0: "got docID=" + docID; + + if (Codec.DEBUG) + System.out.println("PW.addDoc: docID=" + docID + " pendingDocCount=" + pendingDocCount + " vs " + pendingDocs.length + " pulsed=" + pulsed); + + if (!pulsed && pendingDocCount == pendingDocs.length) { + + // OK we just crossed the threshold, this term should + // now be written with our wrapped codec: + wrappedDocsWriter.startTerm(); + + if (Codec.DEBUG) + System.out.println(" now flush buffer"); + + // Flush all buffered docs + for(int i=0;iNOTE: block sizes could be variable */ +public abstract class IntIndexOutput { + /** Write an int to the primary file */ + public abstract void write(int v) throws IOException; + + public abstract static class Index { + + // nocommit + public String desc; + + /** Internally records the current location */ + public abstract void mark() throws IOException; + + /** Copies index from other */ + public abstract void set(Index other) throws IOException; + + /** Writes "location" of current output pointer of primary + * output to different output (out) */ + public abstract void write(IndexOutput indexOut, boolean absolute) throws IOException; + } + + /** If you are indexing the primary output file, call + * this and interact with the returned IndexWriter. */ + public abstract Index index() throws IOException; + + public abstract void close() throws IOException; + + public abstract String descFilePointer() throws IOException; +} \ No newline at end of file Index: src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java (revision 0) @@ -0,0 +1,32 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.BufferedIndexInput; + +import java.io.IOException; + +public abstract class IntStreamFactory { + public IntIndexInput openInput(Directory dir, String fileName) throws IOException { + return openInput(dir, fileName, BufferedIndexInput.BUFFER_SIZE); + } + + public abstract IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException; + public abstract IntIndexOutput createOutput(Directory dir, String fileName) throws IOException; +} Index: src/java/org/apache/lucene/index/codecs/sep/SepCodec.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepCodec.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SepCodec.java (revision 0) @@ -0,0 +1,138 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocsConsumer; +import org.apache.lucene.index.codecs.DocsProducer; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsDictReader; +import org.apache.lucene.index.codecs.standard.StandardTermsDictWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexWriter; +import org.apache.lucene.store.Directory; + +public class SepCodec extends Codec { + + public SepCodec() { + name = "Sep"; + } + + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + + DocsConsumer docsWriter = new SepDocsWriter(state, new SingleIntFactory()); + + boolean success = false; + StandardTermsIndexWriter indexWriter; + try { + indexWriter = new SimpleStandardTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, docsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + docsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + final static String DOC_EXTENSION = "doc"; + final static String SKIP_EXTENSION = "skp"; + final static String FREQ_EXTENSION = "frq"; + final static String POS_EXTENSION = "pos"; + final static String PAYLOAD_EXTENSION = "pyl"; + + public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException { + + DocsProducer docsReader = new SepDocsReader(dir, si, readBufferSize, new SingleIntFactory()); + + StandardTermsIndexReader indexReader; + boolean success = false; + try { + indexReader = new SimpleStandardTermsIndexReader(dir, + fieldInfos, + si.name, + indexDivisor); + success = true; + } finally { + if (!success) { + docsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new StandardTermsDictReader(indexReader, + dir, fieldInfos, si.name, + docsReader, + readBufferSize); + success = true; + return ret; + } finally { + if (!success) { + try { + docsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + public void files(Directory dir, SegmentInfo segmentInfo, Collection files) { + SepDocsReader.files(segmentInfo, files); + StandardTermsDictReader.files(segmentInfo, files); + SimpleStandardTermsIndexReader.files(segmentInfo, files); + } + + public void getExtensions(Collection extensions) { + getSepExtensions(extensions); + } + + public static void getSepExtensions(Collection extensions) { + extensions.add(DOC_EXTENSION); + extensions.add(FREQ_EXTENSION); + extensions.add(SKIP_EXTENSION); + extensions.add(POS_EXTENSION); + extensions.add(PAYLOAD_EXTENSION); + StandardTermsDictReader.getExtensions(extensions); + SimpleStandardTermsIndexReader.getIndexExtensions(extensions); + } +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/index/codecs/sep/SepCodec.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java (revision 0) @@ -0,0 +1,538 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.index.codecs.DocsProducer; +import org.apache.lucene.util.Bits; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.PositionsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.codecs.Codec; + +/** Concrete class that reads the current doc/freq/skip + * postings format */ + +// nocommit -- should we switch "hasProx" higher up? and +// create two separate docs readers, one that also reads +// prox and one that doesn't? + +public class SepDocsReader extends DocsProducer { + + final IntIndexInput freqIn; + final IntIndexInput docIn; + + final IndexInput skipIn; + + IndexInput termsIn; + + private final SepPositionsReader posReader; + + int skipInterval; + int maxSkipLevels; + + public SepDocsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory) throws IOException { + + boolean success = false; + try { + + // nocommit -- freqIn is null if omitTF? + final String frqFileName = IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.FREQ_EXTENSION); + freqIn = intFactory.openInput(dir, frqFileName); + + final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.DOC_EXTENSION); + docIn = intFactory.openInput(dir, docFileName); + + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.SKIP_EXTENSION), readBufferSize); + if (segmentInfo.getHasProx()) { + final String posFileName = IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.POS_EXTENSION); + posReader = new SepPositionsReader(dir, segmentInfo, readBufferSize, intFactory); + } else { + posReader = null; + } + success = true; + } finally { + if (!success) { + close(); + } + } + } + + public static void files(SegmentInfo segmentInfo, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.DOC_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.SKIP_EXTENSION)); + SepPositionsReader.files(segmentInfo, files); + } + + public void start(IndexInput termsIn) throws IOException { + this.termsIn = termsIn; + + // Make sure we are talking to the matching past writer + Codec.checkHeader(termsIn, SepDocsWriter.CODEC, SepPositionsWriter.VERSION_START); + + skipInterval = termsIn.readInt(); + maxSkipLevels = termsIn.readInt(); + if (posReader != null) { + posReader.start(termsIn); + } + } + + public Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException { + + final SepPositionsReader.TermsDictReader posReader2; + if (posReader != null && !fieldInfo.omitTermFreqAndPositions) { + posReader2 = (SepPositionsReader.TermsDictReader) posReader.reader(fieldInfo, termsIn); + } else { + posReader2 = null; + } + + return new TermsDictReader(fieldInfo, posReader2, termsIn); + } + + public void close() throws IOException { + try { + if (freqIn != null) + freqIn.close(); + } finally { + try { + if (docIn != null) + docIn.close(); + } finally { + try { + if (skipIn != null) + skipIn.close(); + } finally { + if (posReader != null) + posReader.close(); + } + } + } + } + + class TermsDictReader extends Reader { + + final IndexInput termsIn; + final FieldInfo fieldInfo; + final IntIndexInput.Reader freqIn; + final IntIndexInput.Index freqIndex; + final IntIndexInput.Reader docIn; + final IntIndexInput.Index docIndex; + final private boolean omitTF; + + long skipOffset; + int docFreq; + + // TODO: abstraction violation (we are storing this with + // the concrete impl. as the type, not the abstract base + // class) + final SepPositionsReader.TermsDictReader posReader; + private SegmentDocsEnum docs; + + TermsDictReader(FieldInfo fieldInfo, SepPositionsReader.TermsDictReader posReader, IndexInput termsIn) throws IOException { + this.termsIn = termsIn; // not cloned + this.fieldInfo = fieldInfo; + this.posReader = posReader; + this.docIn = SepDocsReader.this.docIn.reader(); + docIndex = SepDocsReader.this.docIn.index(); + omitTF = fieldInfo.omitTermFreqAndPositions; + if (!omitTF) { + this.freqIn = SepDocsReader.this.freqIn.reader(); + freqIndex = SepDocsReader.this.freqIn.index(); + } else { + this.freqIn = null; + freqIndex = null; + docFreq = 1; + } + } + + public void readTerm(int docFreq, boolean isIndexTerm) throws IOException { + + this.docFreq = docFreq; + if (Codec.DEBUG) { + System.out.println(" dr.readTerm termsFP=" + termsIn.getFilePointer() + " df=" + docFreq + " isIndex=" + isIndexTerm); + System.out.println(" start freqFP=" + freqIndex + " docFP=" + docIndex + " skipFP=" + skipOffset); + } + + if (!omitTF) { + freqIndex.read(termsIn, isIndexTerm); + } + + docIndex.read(termsIn, isIndexTerm); + + if (isIndexTerm) { + skipOffset = termsIn.readVLong(); + } else { + if (docFreq >= skipInterval) { + skipOffset += termsIn.readVLong(); + } + } + + if (Codec.DEBUG) { + System.out.println(" freqFP=" + freqIndex + " docFP=" + docIndex + " skipFP=" + skipOffset); + } + + if (posReader != null) { + posReader.readTerm(docFreq, isIndexTerm); + } + } + + public DocsEnum docs(Bits skipDocs) throws IOException { + + if (docs == null) { + // Lazy init + docs = new SegmentDocsEnum(); + } + + docs.init(skipDocs); + + return docs; + } + + class SegmentDocsEnum extends DocsEnum { + int docFreq; + int doc; + int count; + int freq; + long freqStart; + + // nocommit -- should we do omitTF with 2 different enum classes? + final boolean omitTF; + private Bits skipDocs; + + // nocommit -- should we do hasProx with 2 different enum classes? + + boolean skipped; + SepSkipListReader skipper; + + // TODO: abstraction violation: we are storing the + // concrete impl, not the abstract base class + SepPositionsReader.TermsDictReader.SegmentPositionsEnum positions; + + SegmentDocsEnum() { + if (Codec.DEBUG) { + System.out.println("new docs enum"); + } + omitTF = fieldInfo.omitTermFreqAndPositions; + if (omitTF) { + freq = 1; + } + } + + void init(Bits skipDocs) throws IOException { + if (Codec.DEBUG) { + System.out.println("[" + desc + "] dr.init freqIn seek " + freqIndex + " this=" + this + " (in=" + freqIn + "; this=" + this + ")"); + } + this.skipDocs = skipDocs; + + // nocommit: can't we only do this if consumer + // skipped consuming the previous docs? + docIndex.seek(docIn); + + if (!omitTF) { + freqIndex.seek(freqIn); + } + this.docFreq = TermsDictReader.this.docFreq; + count = 0; + doc = 0; + skipped = false; + proxSkipFreq = 0; + + // maybe not necessary? + proxSkipPayloadLength = -1; + + // TODO: abstraction violation + if (posReader != null) { + //posIndex = posReader.posIndex; + posIndex = posReader.getPosIn().index(); + posIndex.set(posReader.posIndex); + payloadOffset = posReader.payloadOffset; + } + } + + public int next() throws IOException { + + if (Codec.DEBUG) { + if (!omitTF) { + System.out.println("sdr [" + desc + "] next count=" + count + " vs df=" + docFreq + " freqFP=" + freqIn.descFilePointer() + " docFP=" + docIn.descFilePointer() + " skipDocs?=" + (skipDocs != null) ); + } else { + System.out.println("sdr [" + desc + "] next count=" + count + " vs df=" + docFreq + " docFP=" + docIn.descFilePointer() + " skipDocs?=" + (skipDocs != null) ); + } + } + + while(true) { + if (count == docFreq) { + return NO_MORE_DOCS; + } + + count++; + + // Decode next doc + doc += docIn.next(); + + if (!omitTF) { + freq = freqIn.next(); + if (positions != null) { + positions.seek(freq); + } else { + proxSkipFreq += freq; + } + } + + if (Codec.DEBUG) { + System.out.println(" decode doc=" + doc + " freq=" + freq); + } + + if (skipDocs == null || !skipDocs.get(doc)) { + break; + } else if (Codec.DEBUG) { + System.out.println(" doc=" + doc + " is skipped"); + } + } + + // nocommit + if (Codec.DEBUG) { + if (positions != null) { + positions.desc = desc + ":" + doc; + } + System.out.println(" return doc=" + doc); + } + return doc; + } + + public int read(int[] docs, int[] freqs) throws IOException { + // nocommit -- switch to bulk read api in IntIndexInput + int i = 0; + final int length = docs.length; + while (i < length && count < docFreq) { + count++; + // manually inlined call to next() for speed + doc += docIn.next(); + if (!omitTF) { + freq = freqIn.next(); + if (positions != null) { + positions.seek(freq); + } else { + proxSkipFreq += freq; + } + } + + if (skipDocs == null || !skipDocs.get(doc)) { + docs[i] = doc; + freqs[i] = freq; + i++; + } + } + + return i; + } + + public int freq() { + return freq; + } + + // Holds pending seek data for positions: + IntIndexInput.Index posIndex; + long payloadOffset; + int proxSkipPayloadLength; + + // If we step through docs w/o getting positions for + // them, we accumulate how many freqs we've skipped + // here. Then, when positions() is called, we skip + // this many positions to catch up: + int proxSkipFreq; + + PositionsEnum fakePositions; + + public PositionsEnum positions() throws IOException { + + if (Codec.DEBUG) { + System.out.println("sep.positions pos=" + positions + " freq=" + freq); + } + + if (positions == null) { + + // First time positions is requested from this DocsEnum + + // Lazy init + if (posReader == null) { + + // nocommit -- should we return null? + + // TermFreq was omitted from this field during + // indexing, which means we pretend termFreq is + // always 1 with that 1 occurrence having + // position 0 + if (fakePositions == null) { + fakePositions = new FakePositionsEnum(); + } + if (Codec.DEBUG) { + System.out.println(" return fake"); + } + return fakePositions; + } else { + + // nocommit: abstraction violation + positions = (SepPositionsReader.TermsDictReader.SegmentPositionsEnum) posReader.positions(); + if (Codec.DEBUG) { + System.out.println("pos skip posIndex=" + posIndex + " payloadlen=" + proxSkipPayloadLength + " skipPosCount= " + proxSkipFreq); + } + positions.seek(posIndex, payloadOffset, proxSkipPayloadLength); + + // TODO: technically, if this positions is deep + // into the DocsEnum iteration, it'd pay to use + // the skipper to catch up, instead of linear + // scan: + positions.seek(proxSkipFreq); + proxSkipFreq = 0; + } + } + + if (Codec.DEBUG) { + positions.desc = desc + ":" + doc; + } + + positions.catchUp(freq); + + return positions; + } + + public int advance(int target) throws IOException { + + // TODO: jump right to next() if target is < X away + // from where we are now? + + if (Codec.DEBUG) { + System.out.println("sdr [" + desc + "]: advance target=" + target); + } + + if (docFreq >= skipInterval) { + + // There are enough docs in the posting to have + // skip data + if (skipper == null) { + // Lazy init + if (Codec.DEBUG) { + System.out.println(" create skipper"); + } + skipper = new SepSkipListReader((IndexInput) skipIn.clone(), + omitTF ? null : SepDocsReader.this.freqIn, + SepDocsReader.this.docIn, + posReader == null ? null : posReader.getPosIn(), + maxSkipLevels, skipInterval); + } + + if (!skipped) { + + // We haven't yet skipped for this posting, + // so now we init the skipper + + // TODO: this is abstraction violation; instead, + // skipper should interact with this as a + // private consumer + skipper.init(skipOffset, + docIndex, + freqIndex, + posReader != null ? posReader.posIndex : null, + payloadOffset, + docFreq, + fieldInfo.storePayloads); + + if (Codec.DEBUG) { + System.out.println(" init skipper: base skipFP=" + skipOffset + " docFP=" + docIndex + " freqFP=" + freqIndex + " proxFP=" + + (posReader != null ? posReader.posIndex : null) + " payloadFP=" + payloadOffset); + } + + skipped = true; + } + + final int newCount = skipper.skipTo(target); + + if (newCount > count) { + + if (Codec.DEBUG) { + System.out.println("sdr [" + desc + "]: skipper moved to newCount=" + newCount + + " docFP=" + skipper.getDocIndex() + + " freqFP=" + skipper.getFreqIndex() + + " posFP=" + skipper.getPosIndex() + + " payloadFP=" + skipper.getPayloadPointer() + + " doc=" + skipper.getDoc()); + } + + // Skipper did move + if (!omitTF) { + skipper.getFreqIndex().seek(freqIn); + } + skipper.getDocIndex().seek(docIn); + count = newCount; + doc = skipper.getDoc(); + + // TODO: abstraction violation; this should be a + // private interaction b/w skipper & posReader + if (positions != null) { + positions.seek(skipper.getPosIndex(), + skipper.getPayloadPointer(), + skipper.getPayloadLength()); + } else { + if (posIndex != null) { + posIndex.set(skipper.getPosIndex()); + } + payloadOffset = skipper.getPayloadPointer(); + proxSkipPayloadLength = skipper.getPayloadLength(); + proxSkipFreq = 0; + } + } else if (Codec.DEBUG) { + System.out.println(" no skipping to be done"); + } + } + + // Now, linear scan for the rest: + do { + if (next() == NO_MORE_DOCS) { + return NO_MORE_DOCS; + } + } while (target > doc); + + return doc; + } + } + } +} + +/** Returned when someone asks for positions() enum on field + * with omitTf true */ +class FakePositionsEnum extends PositionsEnum { + public int next() { + return 0; + } + public int getPayloadLength() { + return 0; + } + public boolean hasPayload() { + return false; + } + public byte[] getPayload(byte[] data, int offset) { + return null; + } +} Property changes on: src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/sep/SepDocsWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepDocsWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SepDocsWriter.java (revision 0) @@ -0,0 +1,246 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more +u * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.codecs.DocsConsumer; +import org.apache.lucene.index.codecs.PositionsConsumer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.codecs.Codec; + +/** Writes frq to .frq, docs to .doc, pos to .pos, payloads + * to .pyl, skip data to .skp */ + +public final class SepDocsWriter extends DocsConsumer { + final static String CODEC = "SepDocFreqSkip"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final IntIndexOutput freqOut; + final IntIndexOutput.Index freqIndex; + + final IntIndexOutput docOut; + final IntIndexOutput.Index docIndex; + + final IndexOutput skipOut; + IndexOutput termsOut; + + final SepPositionsWriter posWriter; + final SepSkipListWriter skipListWriter; + final int skipInterval; + final int maxSkipLevels; + final int totalNumDocs; + + boolean storePayloads; + boolean omitTF; + + // Starts a new term + long lastSkipStart; + + FieldInfo fieldInfo; + + public SepDocsWriter(SegmentWriteState state, IntStreamFactory factory) throws IOException { + super(); + + final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, SepCodec.FREQ_EXTENSION); + state.flushedFiles.add(frqFileName); + freqOut = factory.createOutput(state.directory, frqFileName); + freqIndex = freqOut.index(); + + final String docFileName = IndexFileNames.segmentFileName(state.segmentName, SepCodec.DOC_EXTENSION); + state.flushedFiles.add(docFileName); + docOut = factory.createOutput(state.directory, docFileName); + docIndex = docOut.index(); + + final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, SepCodec.SKIP_EXTENSION); + state.flushedFiles.add(skipFileName); + skipOut = state.directory.createOutput(skipFileName); + + if (Codec.DEBUG) { + System.out.println("dw.init: create frq=" + frqFileName + " doc=" + docFileName + " skip=" + skipFileName); + } + + totalNumDocs = state.numDocs; + + // nocommit -- abstraction violation + skipListWriter = new SepSkipListWriter(state.skipInterval, + state.maxSkipLevels, + state.numDocs, + freqOut, docOut, + null, null); + + skipInterval = state.skipInterval; + maxSkipLevels = state.maxSkipLevels; + + posWriter = new SepPositionsWriter(state, this, factory); + } + + public void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + Codec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + // nocommit -- just ask skipper to "start" here + termsOut.writeInt(skipInterval); // write skipInterval + termsOut.writeInt(maxSkipLevels); // write maxSkipLevels + posWriter.start(termsOut); + } + + public void startTerm() throws IOException { + docIndex.mark(); + if (!omitTF) { + freqIndex.mark(); + posWriter.startTerm(); + } + skipListWriter.resetSkip(docIndex, freqIndex, posWriter.posIndex); + } + + // nocommit -- should we NOT reuse across fields? would + // be cleaner + + // Currently, this instance is re-used across fields, so + // our parent calls setField whenever the field changes + public void setField(FieldInfo fieldInfo) { + this.fieldInfo = fieldInfo; + omitTF = fieldInfo.omitTermFreqAndPositions; + skipListWriter.setOmitTF(omitTF); + storePayloads = fieldInfo.storePayloads; + posWriter.setField(fieldInfo); + } + + int lastDocID; + int df; + + int count; + + /** Adds a new doc in this term. If this returns null + * then we just skip consuming positions/payloads. */ + public PositionsConsumer addDoc(int docID, int termDocFreq) throws IOException { + + final int delta = docID - lastDocID; + + if (Codec.DEBUG) { + System.out.println(" dw.addDoc [" + desc + "] count=" + (count++) + " docID=" + docID + " lastDocID=" + lastDocID + " delta=" + delta + " omitTF=" + omitTF + " freq=" + termDocFreq); + } + + if (docID < 0 || (df > 0 && delta <= 0)) { + throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); + } + + if ((++df % skipInterval) == 0) { + // TODO: abstraction violation + // nocommit -- awkward we have to make these two + // separate calls to skipper + skipListWriter.setSkipData(lastDocID, storePayloads, posWriter.lastPayloadLength); + skipListWriter.bufferSkip(df); + + if (Codec.DEBUG) { + System.out.println(" bufferSkip lastDocID=" + lastDocID + + " df=" + df + + " docFP=" + docOut.descFilePointer() + + " freqFP=" + freqOut.descFilePointer() + + " posFP=" + posWriter.posOut.descFilePointer() + + " payloadFP=" + skipListWriter.payloadOutput.getFilePointer() + + " payloadLen=" + posWriter.lastPayloadLength); + } + } + + lastDocID = docID; + docOut.write(delta); + if (!omitTF) { + freqOut.write(termDocFreq); + } + + // nocommit + if (Codec.DEBUG) { + ((SepPositionsWriter) posWriter).desc = desc + ":" + docID; + } + + if (omitTF) { + return null; + } else { + return posWriter; + } + } + + /** Called when we are done adding docs to this term */ + public void finishTerm(int docCount, boolean isIndexTerm) throws IOException { + + long skipPos = skipOut.getFilePointer(); + + // nocommit -- wasteful we are counting this in two places? + assert docCount == df; + if (Codec.DEBUG) { + System.out.println("dw.finishTerm termsFP=" + termsOut.getFilePointer() + " df=" + df + " skipPos=" + skipPos); + } + + if (!omitTF) { + freqIndex.write(termsOut, isIndexTerm); + } + docIndex.write(termsOut, isIndexTerm); + + if (df >= skipInterval) { + if (Codec.DEBUG) { + System.out.println(" writeSkip skipPos=" + skipPos + " lastSkipPos=" + lastSkipStart); + } + + skipListWriter.writeSkip(skipOut); + } + + if (isIndexTerm) { + termsOut.writeVLong(skipPos); + lastSkipStart = skipPos; + } else if (df >= skipInterval) { + termsOut.writeVLong(skipPos-lastSkipStart); + lastSkipStart = skipPos; + } + + if (!omitTF) { + posWriter.finishTerm(isIndexTerm); + } + + lastDocID = 0; + df = 0; + + // nocommit + count = 0; + } + + public void close() throws IOException { + if (Codec.DEBUG) + System.out.println("dw.close skipFP=" + skipOut.getFilePointer()); + try { + freqOut.close(); + } finally { + try { + docOut.close(); + } finally { + try { + skipOut.close(); + } finally { + posWriter.close(); + } + } + } + } +} Property changes on: src/java/org/apache/lucene/index/codecs/sep/SepDocsWriter.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java (revision 0) @@ -0,0 +1,308 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PositionsEnum; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PositionsProducer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; + +public class SepPositionsReader extends PositionsProducer { + + final IntIndexInput posIn; + + final IndexInput payloadIn; + + IndexInput termsIn; + + public SepPositionsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory) throws IOException { + assert segmentInfo.getHasProx(); + boolean success = false; + try { + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.POS_EXTENSION), readBufferSize); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.PAYLOAD_EXTENSION), readBufferSize); + success = true; + } finally { + if (!success) { + close(); + } + } + } + + public void start(IndexInput termsIn) throws IOException { + this.termsIn = termsIn; + + // nocomit -- move these 2 constants into XXXCodec? + Codec.checkHeader(termsIn, SepPositionsWriter.CODEC, SepPositionsWriter.VERSION_START); + } + + static void files(SegmentInfo segmentInfo, Collection files) { + if (segmentInfo.getHasProx()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.POS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, SepCodec.PAYLOAD_EXTENSION)); + } + } + + public Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException { + return new TermsDictReader(termsIn, fieldInfo); + } + + public void close() throws IOException { + try { + if (posIn != null) + posIn.close(); + } finally { + if (payloadIn != null) + payloadIn.close(); + } + } + + class TermsDictReader extends Reader { + + final IndexInput termsIn; + final IntIndexInput.Reader posIn; + final IntIndexInput.Index posIndex; + + final FieldInfo fieldInfo; + long payloadOffset; + + TermsDictReader(IndexInput termsIn, FieldInfo fieldInfo) throws IOException { + this.termsIn = termsIn; + this.fieldInfo = fieldInfo; + this.posIn = SepPositionsReader.this.posIn.reader(); + posIndex = SepPositionsReader.this.posIn.index(); + } + + public IntIndexInput getPosIn() { + return SepPositionsReader.this.posIn; + } + + public void readTerm(int docFreq, boolean isIndexTerm) throws IOException { + if (Codec.DEBUG) { + System.out.println(" pr.readterm termsInPointer=" + termsIn.getFilePointer() + " isIndex=" + isIndexTerm); + } + posIndex.read(termsIn, isIndexTerm); + if (isIndexTerm) { + payloadOffset = termsIn.readVLong(); + } else { + payloadOffset += termsIn.readVLong(); + } + if (Codec.DEBUG) { + System.out.println(" posIndex=" + posIndex + " payloadOffset=" + payloadOffset); + } + if (positions != null) { + positions.seek(posIndex, payloadOffset, -1); + } + } + + SegmentPositionsEnum positions; + + public PositionsEnum positions() throws IOException { + + if (positions == null) { + // Lazy init + positions = new SegmentPositionsEnum(posIndex, payloadOffset); + } + + return positions; + } + + // nocommit -- should we have different reader for + // payload vs no payload? + class SegmentPositionsEnum extends PositionsEnum { + + // nocommit + String desc; + + //final IntIndexInput posIn; + final IndexInput payloadIn; + final IntIndexInput.Index pendingPosIndex; + + final boolean storePayloads; + + boolean payloadPending; // True if we must skip payload beore reading next position + + long payloadOffset; + + int position; + int payloadLength; + int posSkipCount; + + private boolean seekPending; + + SegmentPositionsEnum(IntIndexInput.Index posIndex, long payloadOffset) throws IOException { + //posIn = SepPositionsReader.this.posIn.reader(); + this.payloadOffset = payloadOffset; + pendingPosIndex = SepPositionsReader.this.posIn.index(); + pendingPosIndex.set(posIndex); + seekPending = true; + + if (Codec.DEBUG) { + System.out.println("new pos enum seekPending=true posIndex=" + pendingPosIndex); + } + storePayloads = fieldInfo.storePayloads; + if (storePayloads) { + payloadIn = (IndexInput) SepPositionsReader.this.payloadIn.clone(); + } else { + payloadIn = null; + } + } + + public void seek(IntIndexInput.Index posIndex, long payloadOffset, int payloadLength) { + if (Codec.DEBUG) { + System.out.println("spr.seek posIndex=" + posIndex); + } + pendingPosIndex.set(posIndex); + this.payloadOffset = payloadOffset; + this.payloadLength = payloadLength; + posSkipCount = 0; + seekPending = true; + } + + // Cumulative on top of a previons Index seek + public void seek(int posCount) { + posSkipCount += posCount; + if (Codec.DEBUG) { + System.out.println("pr [" + desc + "] skip " + posCount + " positions; now " + posSkipCount); + } + } + + void catchUp(int currentCount) throws IOException { + if (Codec.DEBUG) { + System.out.println("pos catchup [" + desc + "]: seekPending=" + seekPending + " seekPosIndex=" + pendingPosIndex + " payloadPending=" + payloadPending + " payloadFP=" + payloadOffset + " skipPosCount " + posSkipCount + " vs currentCount " + currentCount); + } + + if (seekPending) { + pendingPosIndex.seek(posIn); + if (storePayloads) { + payloadIn.seek(payloadOffset); + } + payloadPending = false; + seekPending = false; + } + + while(posSkipCount > currentCount) { + next(); + } + + if (Codec.DEBUG) { + System.out.println(" pos catchup done"); + } + position = 0; + } + + public int next() throws IOException { + + if (Codec.DEBUG) { + System.out.println("pr.next [" + desc + "]: posFP=" + posIn.descFilePointer() + getPayloadFP()); + } + + final int code = posIn.next(); + + if (storePayloads) { + + if (payloadPending && payloadLength > 0) { + if (Codec.DEBUG) { + System.out.println(" payload pending: skip " + payloadLength + " bytes"); + } + // nocommit: do this lazily, when getPayload() + // is called + payloadIn.seek(payloadIn.getFilePointer()+payloadLength); + } + + if ((code & 1) != 0) { + // Payload length has changed + payloadLength = posIn.next(); + assert payloadLength >= 0; + if (Codec.DEBUG) { + System.out.println(" new payloadLen=" + payloadLength); + } + } + assert payloadLength != -1; + + payloadPending = true; + position += code >>> 1; + } else { + position += code; + } + + posSkipCount--; + + // NOTE: the old API actually allowed this... and some tests actually did it + assert posSkipCount >= 0: "next() was called too many times (more than FormatPostingsDocsEnum.freq() times)"; + + if (Codec.DEBUG) { + System.out.println(" proxFP=" + posIn.descFilePointer() + getPayloadFP() + " return pos=" + position); + } + + return position; + } + + // debugging only + private String getPayloadFP() { + if (payloadIn != null) { + return " payloadFP=" + payloadIn.getFilePointer(); + } else { + return " payloadFP=null"; + } + } + + public int getPayloadLength() { + return payloadLength; + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + + if (!payloadPending) { + throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); + } + + if (Codec.DEBUG) { + System.out.println(" getPayload payloadFP=" + payloadIn.getFilePointer() + " len=" + payloadLength); + } + + final byte[] retArray; + final int retOffset; + if (data == null || data.length-offset < payloadLength) { + // the array is too small to store the payload data, + // so we allocate a new one + retArray = new byte[payloadLength]; + retOffset = 0; + } else { + retArray = data; + retOffset = offset; + } + + payloadIn.readBytes(retArray, retOffset, payloadLength); + payloadPending = false; + return retArray; + } + + public boolean hasPayload() { + return payloadPending && payloadLength > 0; + } + } + } +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java (revision 0) @@ -0,0 +1,195 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.codecs.PositionsConsumer; +import org.apache.lucene.index.codecs.Codec; + +public final class SepPositionsWriter extends PositionsConsumer { + + final static String CODEC = "SepPositionsPayloads"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final SepDocsWriter parent; + final IntIndexOutput posOut; + final IntIndexOutput.Index posIndex; + final IndexOutput payloadOut; + + IndexOutput termsOut; + + boolean omitTF; + boolean storePayloads; + int lastPayloadLength = -1; + + // nocommit + String desc; + + public SepPositionsWriter(SegmentWriteState state, SepDocsWriter parent, IntStreamFactory factory) throws IOException { + this.parent = parent; + omitTF = parent.omitTF; + if (Codec.DEBUG) { + System.out.println("spw.create seg=" + state.segmentName + " dir=" + state.directory); + } + if (state.fieldInfos.hasProx()) { + // At least one field does not omit TF, so create the + + // prox file + final String proxFileName = IndexFileNames.segmentFileName(state.segmentName, SepCodec.POS_EXTENSION); + posOut = factory.createOutput(state.directory, proxFileName); + state.flushedFiles.add(proxFileName); + posIndex = posOut.index(); + + // nocommit -- only if at least one field stores + // payloads? + boolean success = false; + final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, SepCodec.PAYLOAD_EXTENSION); + try { + payloadOut = state.directory.createOutput(payloadFileName); + success = true; + } finally { + if (!success) { + posOut.close(); + } + } + state.flushedFiles.add(payloadFileName); + + if (Codec.DEBUG) { + System.out.println(" hasProx create pos=" + proxFileName + " payload=" + payloadFileName); + } + + parent.skipListWriter.setPosOutput(posOut); + parent.skipListWriter.setPayloadOutput(payloadOut); + } else { + if (Codec.DEBUG) { + System.out.println(" no prox"); + } + // Every field omits TF so we will write no prox file + posIndex = null; + posOut = null; + payloadOut = null; + } + } + + public void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + Codec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + } + + long payloadStart; + long lastPayloadStart; + + public void startTerm() throws IOException { + posIndex.mark(); + payloadStart = payloadOut.getFilePointer(); + lastPayloadLength = -1; + } + + int lastPosition; + + /** Add a new position & payload */ + public void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException { + assert !omitTF: "omitTF is true"; + assert posOut != null; + if (Codec.DEBUG) { + if (payload != null) { + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " posFP=" + posOut.descFilePointer() + " payloadFP=" + payloadOut.getFilePointer() + " payload=" + payloadLength + " bytes"); + } else { + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " posFP=" + posOut.descFilePointer() + " payloadFP=" + payloadOut.getFilePointer()); + } + } + + final int delta = position - lastPosition; + lastPosition = position; + + if (storePayloads) { + if (Codec.DEBUG) { + System.out.println(" store payload len=" + payloadLength); + } + if (payloadLength != lastPayloadLength) { + if (Codec.DEBUG) { + System.out.println(" payload len change old=" + lastPayloadLength + " new=" + payloadLength); + } + lastPayloadLength = payloadLength; + // TODO: explore whether we get better compression + // by not storing payloadLength into prox stream? + posOut.write((delta<<1)|1); + posOut.write(payloadLength); + } else { + posOut.write(delta << 1); + } + + if (payloadLength > 0) { + if (Codec.DEBUG) { + System.out.println(" write @ payloadFP=" + payloadOut.getFilePointer()); + } + payloadOut.writeBytes(payload, payloadLength); + } + } else { + posOut.write(delta); + } + } + + void setField(FieldInfo fieldInfo) { + omitTF = fieldInfo.omitTermFreqAndPositions; + storePayloads = omitTF ? false : fieldInfo.storePayloads; + } + + /** Called when we are done adding positions & payloads */ + public void finishDoc() { + lastPosition = 0; + } + + public void finishTerm(boolean isIndexTerm) throws IOException { + assert !omitTF; + + if (Codec.DEBUG) { + System.out.println("poswriter finishTerm isIndex=" + isIndexTerm + " pointer=" + termsOut.getFilePointer()); + } + + posIndex.write(termsOut, isIndexTerm); + if (isIndexTerm) { + // Write absolute at seek points + termsOut.writeVLong(payloadStart); + } else { + termsOut.writeVLong(payloadStart-lastPayloadStart); + } + + lastPayloadStart = payloadStart; + } + + public void close() throws IOException { + try { + if (posOut != null) { + posOut.close(); + } + } finally { + if (payloadOut != null) { + payloadOut.close(); + } + } + } +} Property changes on: src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java (revision 0) @@ -0,0 +1,231 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.MultiLevelSkipListReader; + +/** + * Implements the skip list reader for the default posting list format + * that stores positions and payloads. + */ + +// TODO: rewrite this as recursive classes? +class SepSkipListReader extends MultiLevelSkipListReader { + private boolean currentFieldStoresPayloads; + private IntIndexInput.Index freqIndex[]; + private IntIndexInput.Index docIndex[]; + private IntIndexInput.Index posIndex[]; + private long payloadPointer[]; + private int payloadLength[]; + + private final IntIndexInput.Index lastFreqIndex; + private final IntIndexInput.Index lastDocIndex; + // nocommit -- make private again + final IntIndexInput.Index lastPosIndex; + + private long lastFreqPointer; + private long lastDocPointer; + private long lastPosPointer; + private long lastPayloadPointer; + private int lastPayloadLength; + private int lastChildLevel; + + SepSkipListReader(IndexInput skipStream, + IntIndexInput freqIn, + IntIndexInput docIn, + IntIndexInput posIn, + int maxSkipLevels, + int skipInterval) + throws IOException { + super(skipStream, maxSkipLevels, skipInterval); + if (freqIn != null) { + freqIndex = new IntIndexInput.Index[maxSkipLevels]; + } + docIndex = new IntIndexInput.Index[maxSkipLevels]; + if (posIn != null) { + posIndex = new IntIndexInput.Index[maxNumberOfSkipLevels]; + } + for(int i=0;i DocSkip, FreqSkip, ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // DocSkip records the document number before every SkipInterval th document in TermFreqs. + // Document numbers are represented as differences from the previous value in the sequence. + // Case 2: current field stores payloads + // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // PayloadLength --> VInt + // In this case DocSkip/2 is the difference between + // the current and the previous value. If DocSkip + // is odd, then a PayloadLength encoded as VInt follows, + // if DocSkip is even, then it is assumed that the + // current payload length equals the length at the previous + // skip point + if (Codec.DEBUG) { + System.out.println("ssw level=" + level + " curDoc=" + curDoc + " lastDoc=" + lastSkipDoc[level] + " delta=" + (curDoc - lastSkipDoc[level]) + " storePayloads=" + curStorePayloads + " skipBufferFP=" + skipBuffer.getFilePointer() + " curPayloadLen=" + curPayloadLength + " freqIndex=" + freqOutput.descFilePointer() + " docIndex=" + docOutput.descFilePointer() + " posIndex=" + posOutput.descFilePointer() + " curPayloadPointer=" + curPayloadPointer); + } + + assert !omitTF || !curStorePayloads; + + if (curStorePayloads) { + int delta = curDoc - lastSkipDoc[level]; + if (curPayloadLength == lastSkipPayloadLength[level]) { + // the current payload length equals the length at the previous skip point, + // so we don't store the length again + skipBuffer.writeVInt(delta << 1); + } else { + // the payload length is different from the previous one. We shift the DocSkip, + // set the lowest bit and store the current payload length as VInt. + skipBuffer.writeVInt(delta << 1 | 1); + skipBuffer.writeVInt(curPayloadLength); + lastSkipPayloadLength[level] = curPayloadLength; + } + } else { + // current field does not store payloads + skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); + } + + if (!omitTF) { + freqIndex[level].mark(); + freqIndex[level].write(skipBuffer, false); + } + docIndex[level].mark(); + docIndex[level].write(skipBuffer, false); + if (!omitTF) { + posIndex[level].mark(); + posIndex[level].write(skipBuffer, false); + skipBuffer.writeVInt((int) (curPayloadPointer - lastSkipPayloadPointer[level])); + } + + lastSkipDoc[level] = curDoc; + lastSkipPayloadPointer[level] = curPayloadPointer; + } +} Property changes on: src/java/org/apache/lucene/index/codecs/sep/SepSkipListWriter.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/sep/SingleIntFactory.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SingleIntFactory.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SingleIntFactory.java (revision 0) @@ -0,0 +1,30 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Directory; +import java.io.IOException; + +public class SingleIntFactory extends IntStreamFactory { + public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { + return new SingleIntIndexInput(dir, fileName, readBufferSize); + } + public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { + return new SingleIntIndexOutput(dir, fileName); + } +} Index: src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java (revision 0) @@ -0,0 +1,112 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.Codec; +import java.io.IOException; + +/** Reads IndexInputs written with {@link + * SingleIntIndexoutput} */ +public class SingleIntIndexInput extends IntIndexInput { + private final IndexInput in; + + public SingleIntIndexInput(Directory dir, String fileName, int readBufferSize) + throws IOException { + in = dir.openInput(fileName, readBufferSize); + Codec.checkHeader(in, SingleIntIndexOutput.CODEC, SingleIntIndexOutput.VERSION_START); + } + + public Reader reader() throws IOException { + return new Reader((IndexInput) in.clone()); + } + + public void close() throws IOException { + in.close(); + } + + public static class Reader extends IntIndexInput.Reader { + // clone: + private final IndexInput in; + + private final BulkReadResult result = new BulkReadResult(); + + public Reader(IndexInput in) { + this.in = in; + result.offset = 0; + } + + /** Reads next single int */ + public int next() throws IOException { + return in.readVInt(); + } + + /** Reads next chunk of ints */ + public BulkReadResult read(int[] buffer, int count) throws IOException { + result.buffer = buffer; + for(int i=0;i>>= 1; + } else { + delta = skipStream.readVInt(); + } + freqPointer[level] += skipStream.readVInt(); + proxPointer[level] += skipStream.readVInt(); + + return delta; + } +} Index: src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java (revision 0) @@ -0,0 +1,149 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.MultiLevelSkipListWriter; + + +/** + * Implements the skip list writer for the default posting list format + * that stores positions and payloads. + * + */ +// nocommit -- made public +public class DefaultSkipListWriter extends MultiLevelSkipListWriter { + private int[] lastSkipDoc; + private int[] lastSkipPayloadLength; + private long[] lastSkipFreqPointer; + private long[] lastSkipProxPointer; + + private IndexOutput freqOutput; + // nocommit -- private again + public IndexOutput proxOutput; + + private int curDoc; + private boolean curStorePayloads; + private int curPayloadLength; + private long curFreqPointer; + private long curProxPointer; + + // nocommit made public + public DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput) { + super(skipInterval, numberOfSkipLevels, docCount); + this.freqOutput = freqOutput; + this.proxOutput = proxOutput; + + lastSkipDoc = new int[numberOfSkipLevels]; + lastSkipPayloadLength = new int[numberOfSkipLevels]; + lastSkipFreqPointer = new long[numberOfSkipLevels]; + lastSkipProxPointer = new long[numberOfSkipLevels]; + } + + // nocommit -- made public + public void setFreqOutput(IndexOutput freqOutput) { + this.freqOutput = freqOutput; + } + + // nocommit -- made public + public void setProxOutput(IndexOutput proxOutput) { + this.proxOutput = proxOutput; + } + + /** + * Sets the values for the current skip data. + */ + // nocommit -- made public + public void setSkipData(int doc, boolean storePayloads, int payloadLength) { + this.curDoc = doc; + this.curStorePayloads = storePayloads; + this.curPayloadLength = payloadLength; + this.curFreqPointer = freqOutput.getFilePointer(); + if (proxOutput != null) + this.curProxPointer = proxOutput.getFilePointer(); + } + + // nocommit -- made public + public void resetSkip() { + super.resetSkip(); + Arrays.fill(lastSkipDoc, 0); + Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list + Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); + if (proxOutput != null) + Arrays.fill(lastSkipProxPointer, proxOutput.getFilePointer()); + if (Codec.DEBUG) { + if (proxOutput != null) + System.out.println(" skip writer base freqFP=" + freqOutput.getFilePointer() + " proxFP=" + proxOutput.getFilePointer()); + else + System.out.println(" skip writer base freqFP=" + freqOutput.getFilePointer()); + } + } + + protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { + // To efficiently store payloads in the posting lists we do not store the length of + // every payload. Instead we omit the length for a payload if the previous payload had + // the same length. + // However, in order to support skipping the payload length at every skip point must be known. + // So we use the same length encoding that we use for the posting lists for the skip data as well: + // Case 1: current field does not store payloads + // SkipDatum --> DocSkip, FreqSkip, ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // DocSkip records the document number before every SkipInterval th document in TermFreqs. + // Document numbers are represented as differences from the previous value in the sequence. + // Case 2: current field stores payloads + // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // PayloadLength --> VInt + // In this case DocSkip/2 is the difference between + // the current and the previous value. If DocSkip + // is odd, then a PayloadLength encoded as VInt follows, + // if DocSkip is even, then it is assumed that the + // current payload length equals the length at the previous + // skip point + if (curStorePayloads) { + int delta = curDoc - lastSkipDoc[level]; + if (curPayloadLength == lastSkipPayloadLength[level]) { + // the current payload length equals the length at the previous skip point, + // so we don't store the length again + skipBuffer.writeVInt(delta * 2); + } else { + // the payload length is different from the previous one. We shift the DocSkip, + // set the lowest bit and store the current payload length as VInt. + skipBuffer.writeVInt(delta * 2 + 1); + skipBuffer.writeVInt(curPayloadLength); + lastSkipPayloadLength[level] = curPayloadLength; + } + } else { + // current field does not store payloads + skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); + } + skipBuffer.writeVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); + skipBuffer.writeVInt((int) (curProxPointer - lastSkipProxPointer[level])); + + lastSkipDoc[level] = curDoc; + //System.out.println("write doc at level " + level + ": " + curDoc); + + lastSkipFreqPointer[level] = curFreqPointer; + lastSkipProxPointer[level] = curProxPointer; + } + +} Index: src/java/org/apache/lucene/index/codecs/standard/DeltaBytesReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/DeltaBytesReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/DeltaBytesReader.java (revision 0) @@ -0,0 +1,53 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.index.TermRef; + +import java.io.IOException; + +// Handles reading incremental UTF8 encoded terms +final class DeltaBytesReader { + final TermRef term = new TermRef(); + final IndexInput in; + boolean started; + + DeltaBytesReader(IndexInput in) { + this.in = in; + term.bytes = new byte[10]; + } + + void reset(TermRef text) { + term.copy(text); + } + + void read() throws IOException { + // mxx + //System.out.println(Thread.currentThread().getName() + ": dbr termFP=" + in.getFilePointer()); + final int start = in.readVInt(); + final int suffix = in.readVInt(); + // mxx + //System.out.println(Thread.currentThread().getName() + ": start=" + start + " suffix=" + suffix); + assert start <= term.length: "start=" + start + " length=" + term.length; + final int newLength = start+suffix; + term.grow(newLength); + in.readBytes(term.bytes, start, suffix); + term.length = newLength; + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/DeltaBytesReader.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/DeltaBytesWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/DeltaBytesWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/DeltaBytesWriter.java (revision 0) @@ -0,0 +1,64 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; + +final class DeltaBytesWriter { + + private final UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result(); + + private byte[] lastBytes = new byte[10]; + private int lastLength; + final IndexOutput out; + + DeltaBytesWriter(IndexOutput out) { + this.out = out; + } + + void reset() { + lastLength = 0; + } + + void write(byte[] bytes, int length) throws IOException { + int start = 0; + final int limit = length < lastLength ? length : lastLength; + while(start < limit) { + if (bytes[start] != lastBytes[start]) + break; + start++; + } + + final int suffix = length - start; + // mxx + //System.out.println(Thread.currentThread().getName() + ": dbw start=" + start + " suffix=" + suffix + " outFP=" + out.getFilePointer()); + + out.writeVInt(start); // prefix + out.writeVInt(suffix); // suffix + out.writeBytes(bytes, start, suffix); + if (lastBytes.length < bytes.length) { + lastBytes = ArrayUtil.grow(lastBytes, bytes.length); + } + System.arraycopy(bytes, start, lastBytes, start, suffix); + lastLength = length; + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/DeltaBytesWriter.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java (revision 0) @@ -0,0 +1,435 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.util.ArrayUtil; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Collection; +import java.io.IOException; + +/** + * Uses a simplistic format to record terms dict index + * information. Limititations: + * + * - Index for all fields is loaded entirely into RAM up + * front + * - Index is stored in RAM using shared byte[] that + * wastefully expand every term. Using FST to share + * common prefix & suffix would save RAM. + * - Index is taken at regular numTerms (every 128 by + * default); might be better to do it by "net docFreqs" + * encountered, so that for spans of low-freq terms we + * take index less often. + * + * A better approach might be something similar to how + * postings are encoded, w/ multi-level skips. Ie, load all + * terms index data into memory, as a single large compactly + * encoded stream (eg delta bytes + delta offset). Index + * that w/ multi-level skipper. Then to look up a term is + * the equivalent binary search, using the skipper instead, + * while data remains compressed in memory. + */ + +import org.apache.lucene.index.IndexFileNames; + +public class SimpleStandardTermsIndexReader extends StandardTermsIndexReader { + + final private int totalIndexInterval; + final private int indexDivisor; + final private int indexInterval; + + final private IndexInput in; + private volatile boolean indexLoaded; + + final HashMap fields = new HashMap(); + + public SimpleStandardTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor) + throws IOException { + + IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, StandardCodec.TERMS_INDEX_EXTENSION)); + + try { + Codec.checkHeader(in, SimpleStandardTermsIndexWriter.CODEC_NAME, SimpleStandardTermsIndexWriter.VERSION_START); + + if (Codec.DEBUG) { + System.out.println(" readDirStart @ " + in.getFilePointer()); + } + + final long dirOffset = in.readLong(); + + indexInterval = in.readInt(); + this.indexDivisor = indexDivisor; + + if (indexDivisor == -1) { + totalIndexInterval = indexInterval; + } else { + // In case terms index gets loaded, later, on demand + totalIndexInterval = indexInterval * indexDivisor; + } + + // Read directory + in.seek(dirOffset); + + final int numFields = in.readInt(); + + if (Codec.DEBUG) { + System.out.println("sstir create seg=" + segment + " numFields=" + numFields + " dirStart=" + dirOffset); + } + + for(int i=0;i 0: "numIndexTerms=" + numIndexTerms + " indexDivisor=" + indexDivisor; + + if (blocks == null) { + blocks = new byte[1][]; + blocks[0] = new byte[BYTE_BLOCK_SIZE]; + } + + byte[] lastBlock = blocks[blockUpto]; + int lastBlockOffset = blockOffset; + + fileOffset = new long[this.numIndexTerms]; + blockPointer = new long[this.numIndexTerms]; + termLength = new short[this.numIndexTerms]; + + // nocommit: unused? + //final DeltaBytesReader bytesReader = new DeltaBytesReader(clone); + + final byte[] skipBytes; + if (indexDivisor != 1) { + skipBytes = new byte[128]; + } else { + skipBytes = null; + } + + int upto = 0; + long pointer = 0; + + for(int i=0;i BYTE_BLOCK_SIZE) { + // New block + final byte[] newBlock = new byte[BYTE_BLOCK_SIZE]; + if (blocks.length == blockUpto-1) { + final int newSize = ArrayUtil.getNextSize(blockUpto+1); + final byte[][] newBlocks = new byte[newSize][]; + System.arraycopy(blocks, 0, newBlocks, 0, blocks.length); + blocks = newBlocks; + } + blocks[blockUpto] = newBlock; + blockUpto++; + blockOffset = 0; + } + + final byte[] block = blocks[blockUpto]; + + // Copy old prefix + assert lastBlock != null || start == 0; + assert block != null; + System.arraycopy(lastBlock, lastBlockOffset, block, blockOffset, start); + + // Read new suffix + clone.readBytes(block, blockOffset+start, suffix); + + // Advance file offset + pointer += clone.readVLong(); + + assert thisTermLength < Short.MAX_VALUE; + + termLength[upto] = (short) thisTermLength; + fileOffset[upto] = pointer; + blockPointer[upto] = blockUpto * BYTE_BLOCK_SIZE + blockOffset; + TermRef tr = new TermRef(); + tr.bytes = blocks[blockUpto]; + tr.offset = blockOffset; + tr.length = thisTermLength; + //System.out.println(" read index term=" + new String(blocks[blockUpto], blockOffset, thisTermLength, "UTF-8") + " this=" + this + " bytes=" + block + " (vs=" + blocks[blockUpto] + ") offset=" + blockOffset); + //System.out.println(" read index term=" + tr.toBytesString() + " this=" + this + " bytes=" + block + " (vs=" + blocks[blockUpto] + ") offset=" + blockOffset); + + lastBlock = block; + lastBlockOffset = blockOffset; + blockOffset += thisTermLength; + upto++; + } else { + // Skip bytes + int toSkip = suffix; + while(true) { + if (toSkip > skipBytes.length) { + clone.readBytes(skipBytes, 0, skipBytes.length); + toSkip -= skipBytes.length; + } else { + clone.readBytes(skipBytes, 0, toSkip); + break; + } + } + + // Advance file offset + pointer += clone.readVLong(); + } + } + + // nocommit: put in finally clause + clone.close(); + + assert upto == this.numIndexTerms; + + if (Codec.DEBUG) { + System.out.println(" done read"); + } + } + + final private TermRef termBuffer = new TermRef(); + final private TermsIndexResult termsIndexResult = new TermsIndexResult(); + + public final void getIndexOffset(TermRef term, TermsIndexResult result) throws IOException { + + if (Codec.DEBUG) { + System.out.println("getIndexOffset field=" + fieldInfo.name + " term=" + term + " indexLen = " + blockPointer.length + " numIndexTerms=" + fileOffset.length + " this=" + this); + } + + int lo = 0; // binary search + int hi = fileOffset.length - 1; + + while (hi >= lo) { + int mid = (lo + hi) >> 1; + + final long loc = blockPointer[mid]; + result.term.bytes = blocks[(int) (loc >> BYTE_BLOCK_SHIFT)]; + result.term.offset = (int) (loc & BYTE_BLOCK_MASK); + //System.out.println(" cycle mid=" + mid + " bytes=" + result.term.bytes + " offset=" + result.term.offset); + result.term.length = termLength[mid]; + //System.out.println(" term=" + result.term); + + int delta = term.compareTerm(result.term); + if (delta < 0) { + hi = mid - 1; + } else if (delta > 0) { + lo = mid + 1; + } else { + assert mid >= 0; + result.position = mid*totalIndexInterval; + result.offset = fileOffset[mid]; + return; + } + } + if (hi < 0) { + assert hi == -1; + hi = 0; + } + + final long loc = blockPointer[hi]; + result.term.bytes = blocks[(int) (loc >> BYTE_BLOCK_SHIFT)]; + result.term.offset = (int) (loc & BYTE_BLOCK_MASK); + result.term.length = termLength[hi]; + //System.out.println(" hi term=" + result.term); + + result.position = hi*totalIndexInterval; + result.offset = fileOffset[hi]; + return; + } + } + } + + public void loadTermsIndex() throws IOException { + + if (!indexLoaded) { + + // mxx + if (Codec.DEBUG) { + System.out.println(Thread.currentThread().getName() + ": sstir: load coreIndex on demand"); + } + + Iterator it = fields.values().iterator(); + while(it.hasNext()) { + it.next().loadTermsIndex(); + } + indexLoaded = true; + } + } + + public FieldReader getField(FieldInfo fieldInfo) { + return fields.get(fieldInfo); + } + + public static void files(SegmentInfo info, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, StandardCodec.TERMS_INDEX_EXTENSION)); + } + + public static void getIndexExtensions(Collection extensions) { + extensions.add(StandardCodec.TERMS_INDEX_EXTENSION); + } + + public void getExtensions(Collection extensions) { + getIndexExtensions(extensions); + } + + public void close() throws IOException { + if (in != null) { + in.close(); + } + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexWriter.java (revision 0) @@ -0,0 +1,137 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.Codec; + +import java.util.List; +import java.util.ArrayList; +import java.io.IOException; + +public class SimpleStandardTermsIndexWriter extends StandardTermsIndexWriter { + final private IndexOutput out; + + final static String CODEC_NAME = "SIMPLE_STANDARD_TERMS_INDEX"; + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final private int termIndexInterval; + + private final List fields = new ArrayList(); + private final FieldInfos fieldInfos; + private IndexOutput termsOut; + + // nocommit + final private String segment; + + public SimpleStandardTermsIndexWriter(SegmentWriteState state) throws IOException { + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.TERMS_INDEX_EXTENSION); + state.flushedFiles.add(indexFileName); + this.segment = state.segmentName; + termIndexInterval = state.termIndexInterval; + out = state.directory.createOutput(indexFileName); + Codec.writeHeader(out, CODEC_NAME, VERSION_CURRENT); + fieldInfos = state.fieldInfos; + + // Placeholder for dir offset + out.writeLong(0); + out.writeInt(termIndexInterval); + termWriter = new DeltaBytesWriter(out); + } + + @Override + public void setTermsOutput(IndexOutput termsOut) { + this.termsOut = termsOut; + } + + final private DeltaBytesWriter termWriter; + private FieldInfo currentField; + + public FieldWriter addField(FieldInfo field) { + currentField = field; + SimpleFieldWriter writer = new SimpleFieldWriter(field); + fields.add(writer); + return writer; + } + + private class SimpleFieldWriter extends FieldWriter { + final FieldInfo fieldInfo; + int numIndexTerms; + private long lastTermsPointer; + final long indexStart; + private int numTerms; + + SimpleFieldWriter(FieldInfo fieldInfo) { + this.fieldInfo = fieldInfo; + indexStart = out.getFilePointer(); + termWriter.reset(); + } + + public boolean checkIndexTerm(byte[] term, int termLength, int docFreq) throws IOException { + // First term is first indexed term: + if (0 == (numTerms++ % termIndexInterval)) { + final long termsPointer = termsOut.getFilePointer(); + if (Codec.DEBUG) { + System.out.println("sstiw.checkIndexTerm write index field=" + fieldInfo.name + " term=" + new String(term, 0, termLength, "UTF-8") + " termsFP=" + termsPointer + " numIndexTerms=" + numIndexTerms + " outFP=" + out.getFilePointer()); + } + // mxx + //System.out.println(Thread.currentThread().getName() + ": ii seg=" + segment + " term=" + fieldInfo.name + ":" + new String(term, 0, termLength, "UTF-8") + " numTerms=" + (numTerms-1) + " termFP=" + termsPointer); + termWriter.write(term, termLength); + out.writeVLong(termsPointer - lastTermsPointer); + lastTermsPointer = termsPointer; + numIndexTerms++; + return true; + } else { + return false; + } + } + } + + public void close() throws IOException { + final long dirStart = out.getFilePointer(); + if (Codec.DEBUG) { + System.out.println("sstiw.close seg=" + segment + " dirStart=" + dirStart); + } + final int fieldCount = fields.size(); + + out.writeInt(fieldCount); + for(int i=0;i doc); + + return doc; + } + } + } +} + +/** Returned when someone asks for positions() enum on field + * with omitTf true */ +class FormatPostingsFakePositionsEnum extends PositionsEnum { + public int next() { + return 0; + } + public int getPayloadLength() { + return 0; + } + public boolean hasPayload() { + return false; + } + public byte[] getPayload(byte[] data, int offset) { + return null; + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/StandardDocsWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardDocsWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/StandardDocsWriter.java (revision 0) @@ -0,0 +1,205 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Consumes doc & freq, writing them using the current + * index file format */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.codecs.DocsConsumer; +import org.apache.lucene.index.codecs.PositionsConsumer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.codecs.Codec; + +public final class StandardDocsWriter extends DocsConsumer { + final static String CODEC = "SingleFileDocFreqSkip"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final IndexOutput out; + final StandardPositionsWriter posWriter; + final DefaultSkipListWriter skipListWriter; + final int skipInterval; + final int maxSkipLevels; + final int totalNumDocs; + IndexOutput termsOut; + + boolean omitTermFreqAndPositions; + boolean storePayloads; + // Starts a new term + long lastFreqStart; + long freqStart; + FieldInfo fieldInfo; + + public StandardDocsWriter(SegmentWriteState state) throws IOException { + super(); + final String fileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.FREQ_EXTENSION); + state.flushedFiles.add(fileName); + out = state.directory.createOutput(fileName); + totalNumDocs = state.numDocs; + + // nocommit -- abstraction violation + skipListWriter = new DefaultSkipListWriter(state.skipInterval, + state.maxSkipLevels, + state.numDocs, + out, + null); + + skipInterval = state.skipInterval; + maxSkipLevels = state.maxSkipLevels; + + posWriter = new StandardPositionsWriter(state, this); + } + + public void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + Codec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + termsOut.writeInt(skipInterval); // write skipInterval + termsOut.writeInt(maxSkipLevels); // write maxSkipLevels + posWriter.start(termsOut); + } + + public void startTerm() { + freqStart = out.getFilePointer(); + if (!omitTermFreqAndPositions) + posWriter.startTerm(); + skipListWriter.resetSkip(); + } + + // nocommit -- should we NOT reuse across fields? would + // be cleaner + + // Currently, this instance is re-used across fields, so + // our parent calls setField whenever the field changes + public void setField(FieldInfo fieldInfo) { + this.fieldInfo = fieldInfo; + omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions; + storePayloads = fieldInfo.storePayloads; + posWriter.setField(fieldInfo); + } + + int lastDocID; + int df; + + int count; + + /** Adds a new doc in this term. If this returns null + * then we just skip consuming positions/payloads. */ + public PositionsConsumer addDoc(int docID, int termDocFreq) throws IOException { + + final int delta = docID - lastDocID; + + if (Codec.DEBUG) { + System.out.println(" dw.addDoc [" + desc + "] count=" + (count++) + " docID=" + docID + " lastDocID=" + lastDocID + " delta=" + delta + " omitTF=" + omitTermFreqAndPositions + " freq=" + termDocFreq + " freqPointer=" + out.getFilePointer()); + } + + if (docID < 0 || (df > 0 && delta <= 0)) { + throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); + } + + if ((++df % skipInterval) == 0) { + // TODO: abstraction violation + skipListWriter.setSkipData(lastDocID, storePayloads, posWriter.lastPayloadLength); + skipListWriter.bufferSkip(df); + if (Codec.DEBUG) { + System.out.println(" bufferSkip lastDocID=" + lastDocID + " df=" + df + " freqFP=" + out.getFilePointer() + " proxFP=" + skipListWriter.proxOutput.getFilePointer()); + } + } + + // nocommit -- move this assert up above; every consumer + // shouldn't have to check for this bug: + assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; + + lastDocID = docID; + if (omitTermFreqAndPositions) { + out.writeVInt(delta); + } else if (1 == termDocFreq) { + out.writeVInt((delta<<1) | 1); + } else { + out.writeVInt(delta<<1); + out.writeVInt(termDocFreq); + } + + // nocommit + if (Codec.DEBUG) { + ((StandardPositionsWriter) posWriter).desc = desc + ":" + docID; + } + + if (omitTermFreqAndPositions) { + return null; + } else { + return posWriter; + } + } + + /** Called when we are done adding docs to this term */ + public void finishTerm(int docCount, boolean isIndexTerm) throws IOException { + // nocommit -- wasteful we are counting this in two places? + assert docCount == df; + // mxx + if (Codec.DEBUG) { + System.out.println(Thread.currentThread().getName() + ": dw.finishTerm termsOut pointer=" + termsOut.getFilePointer() + " freqStart=" + freqStart + " df=" + df + " isIndex?=" + isIndexTerm); + } + + if (isIndexTerm) { + // Write absolute at seek points + termsOut.writeVLong(freqStart); + } else { + // Write delta between seek points + termsOut.writeVLong(freqStart - lastFreqStart); + } + + lastFreqStart = freqStart; + + if (df >= skipInterval) { + // mxx + if (Codec.DEBUG) { + System.out.println(Thread.currentThread().getName() + ": writeSkip @ freqFP=" + out.getFilePointer() + " freqStartFP=" + freqStart); + } + termsOut.writeVLong(skipListWriter.writeSkip(out)-freqStart); + } + + if (!omitTermFreqAndPositions) { + posWriter.finishTerm(isIndexTerm); + } + + + lastDocID = 0; + df = 0; + + // nocommit + count = 0; + } + + public void close() throws IOException { + if (Codec.DEBUG) + System.out.println("docs writer close pointer=" + out.getFilePointer()); + try { + out.close(); + } finally { + posWriter.close(); + } + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/StandardDocsWriter.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java (revision 0) @@ -0,0 +1,253 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PositionsEnum; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PositionsProducer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; + +// nocommit -- base class should not be named terms dict: +// this class interacts w/ a docsreader +public class StandardPositionsReader extends PositionsProducer { + + final IndexInput proxIn; + IndexInput termsIn; + + public StandardPositionsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize) throws IOException { + assert segmentInfo.getHasProx(); + proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, StandardCodec.PROX_EXTENSION), readBufferSize); + } + + public void start(IndexInput termsIn) throws IOException { + this.termsIn = termsIn; + + Codec.checkHeader(termsIn, StandardPositionsWriter.CODEC, StandardPositionsWriter.VERSION_START); + } + + public static void files(SegmentInfo segmentInfo, Collection files) { + if (segmentInfo.getHasProx()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, StandardCodec.PROX_EXTENSION)); + } + } + + public Reader reader(FieldInfo fieldInfo, IndexInput termsIn) { + return new TermsDictReader(termsIn, fieldInfo); + } + + public void close() throws IOException { + if (proxIn != null) { + proxIn.close(); + } + } + + class TermsDictReader extends Reader { + + final IndexInput termsIn; + final FieldInfo fieldInfo; + long proxOffset; + + TermsDictReader(IndexInput termsIn, FieldInfo fieldInfo) { + this.termsIn = termsIn; + this.fieldInfo = fieldInfo; + } + + public void readTerm(int docFreq, boolean isIndexTerm) throws IOException { + // mxx + if (Codec.DEBUG) { + System.out.println(" pr.readterm termsInPointer=" + termsIn.getFilePointer() + " isIndex=" + isIndexTerm); + } + + if (isIndexTerm) { + proxOffset = termsIn.readVLong(); + } else { + proxOffset += termsIn.readVLong(); + } + + // mxx + if (Codec.DEBUG) { + System.out.println(" proxOffset=" + proxOffset); + } + + if (positions != null) { + positions.seekPending = true; + positions.skipOffset = proxOffset; + positions.skipPosCount = 0; + } + } + + SegmentPositionsEnum positions; + + public PositionsEnum positions() throws IOException { + + if (positions == null) + // Lazy init + positions = new SegmentPositionsEnum(); + + return positions; + } + + // nocommit -- should we have different reader for + // payload vs no payload? + class SegmentPositionsEnum extends PositionsEnum { + + // nocommit + String desc; + + final IndexInput proxIn; + + final boolean storePayloads; + + boolean seekPending; // True if we must seek before reading next position + boolean payloadPending; // True if we must skip payload beore reading next position + + long skipOffset; + int skipPosCount; + + int position; + int payloadLength; + + SegmentPositionsEnum() { + if (Codec.DEBUG) { + System.out.println("new pos enum"); + } + proxIn = (IndexInput) StandardPositionsReader.this.proxIn.clone(); + storePayloads = fieldInfo.storePayloads; + } + + void skip(long proxOffset, int lastPayloadLength, int numPositions) { + skipOffset = proxOffset; + payloadLength = lastPayloadLength; + assert payloadLength >= 0 || payloadLength == -1; + skipPosCount = numPositions; + seekPending = true; + payloadPending = false; + if (Codec.DEBUG) { + System.out.println("pr [" + desc + "] skip fp= " + proxOffset + " numPositions=" + numPositions); + } + } + + void skip(int numPositions) { + skipPosCount += numPositions; + if (Codec.DEBUG) + System.out.println("pr [" + desc + "] skip " + numPositions + " positions; now " + skipPosCount); + } + + void catchUp(int currentCount) throws IOException { + if (Codec.DEBUG) { + System.out.println(" pos catchup: seekPending=" + seekPending + " skipOffset=" + skipOffset + " skipPosCount " + skipPosCount + " vs currentCount " + currentCount + " payloadLen=" + payloadLength); + } + + if (seekPending) { + proxIn.seek(skipOffset); + seekPending = false; + } + + while(skipPosCount > currentCount) { + next(); + } + if (Codec.DEBUG) { + System.out.println(" pos catchup done"); + } + positions.init(); + } + + void init() { + if (Codec.DEBUG) { + System.out.println(" pos init"); + } + position = 0; + } + + public int next() throws IOException { + + if (Codec.DEBUG) + System.out.println(" pr.next [" + desc + "]: fp=" + proxIn.getFilePointer() + " return pos=" + position); + + if (storePayloads) { + + if (payloadPending && payloadLength > 0) { + if (Codec.DEBUG) + System.out.println(" payload pending: skip " + payloadLength + " bytes"); + proxIn.seek(proxIn.getFilePointer()+payloadLength); + } + + final int code = proxIn.readVInt(); + if ((code & 1) != 0) { + // Payload length has changed + payloadLength = proxIn.readVInt(); + assert payloadLength >= 0; + if (Codec.DEBUG) + System.out.println(" new payloadLen=" + payloadLength); + } + assert payloadLength != -1; + + payloadPending = true; + position += code >>> 1; + } else + position += proxIn.readVInt(); + + skipPosCount--; + + // NOTE: the old API actually allowed this... + assert skipPosCount >= 0: "next() was called too many times (more than FormatPostingsDocsEnum.freq() times)"; + + if (Codec.DEBUG) + System.out.println(" proxFP=" + proxIn.getFilePointer() + " return pos=" + position); + return position; + } + + public int getPayloadLength() { + return payloadLength; + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + + if (!payloadPending) + throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); + + final byte[] retArray; + final int retOffset; + if (data == null || data.length-offset < payloadLength) { + // the array is too small to store the payload data, + // so we allocate a new one + retArray = new byte[payloadLength]; + retOffset = 0; + } else { + retArray = data; + retOffset = offset; + } + + proxIn.readBytes(retArray, retOffset, payloadLength); + payloadPending = false; + return retArray; + } + + public boolean hasPayload() { + return payloadPending && payloadLength > 0; + } + } + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java (revision 0) @@ -0,0 +1,151 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PositionsConsumer; +import org.apache.lucene.store.IndexOutput; + +final class StandardPositionsWriter extends PositionsConsumer { + final static String CODEC = "SingleFilePositionsPayloads"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final StandardDocsWriter parent; + final IndexOutput out; + + IndexOutput termsOut; + + boolean omitTermFreqAndPositions; + boolean storePayloads; + int lastPayloadLength = -1; + + // nocommit + String desc; + + StandardPositionsWriter(SegmentWriteState state, StandardDocsWriter parent) throws IOException { + this.parent = parent; + omitTermFreqAndPositions = parent.omitTermFreqAndPositions; + if (state.fieldInfos.hasProx()) { + // At least one field does not omit TF, so create the + // prox file + final String fileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.PROX_EXTENSION); + state.flushedFiles.add(fileName); + out = state.directory.createOutput(fileName); + parent.skipListWriter.setProxOutput(out); + } else + // Every field omits TF so we will write no prox file + out = null; + } + + public void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + Codec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + } + + long proxStart; + long lastProxStart; + + public void startTerm() { + proxStart = out.getFilePointer(); + lastPayloadLength = -1; + } + + + int lastPosition; + + /** Add a new position & payload */ + public void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException { + assert !omitTermFreqAndPositions: "omitTermFreqAndPositions is true"; + assert out != null; + + if (Codec.DEBUG) { + if (payload != null) + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " fp=" + out.getFilePointer() + " payload=" + payloadLength + " bytes"); + else + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " fp=" + out.getFilePointer()); + } + + final int delta = position - lastPosition; + + assert delta > 0 || position == 0 || position == -1: "position=" + position + " lastPosition=" + lastPosition; // not quite right (if pos=0 is repeated twice we don't catch it) + + lastPosition = position; + + if (storePayloads) { + if (Codec.DEBUG) { + System.out.println(" store payloads"); + } + + if (payloadLength != lastPayloadLength) { + if (Codec.DEBUG) { + System.out.println(" payload len change old=" + lastPayloadLength + " new=" + payloadLength); + } + + lastPayloadLength = payloadLength; + out.writeVInt((delta<<1)|1); + out.writeVInt(payloadLength); + } else + out.writeVInt(delta << 1); + if (payloadLength > 0) + out.writeBytes(payload, payloadLength); + } else + out.writeVInt(delta); + } + + void setField(FieldInfo fieldInfo) { + omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions; + storePayloads = omitTermFreqAndPositions ? false : fieldInfo.storePayloads; + } + + /** Called when we are done adding positions & payloads */ + public void finishDoc() { + lastPosition = 0; + } + + public void finishTerm(boolean isIndexTerm) throws IOException { + assert !omitTermFreqAndPositions; + + // mxx + if (Codec.DEBUG) { + System.out.println("poswriter finishTerm isIndex=" + isIndexTerm + " proxStart=" + proxStart + " pointer=" + termsOut.getFilePointer()); + } + + if (isIndexTerm) { + // Write absolute at seek points + termsOut.writeVLong(proxStart); + } else { + termsOut.writeVLong(proxStart-lastProxStart); + } + + lastProxStart = proxStart; + } + + public void close() throws IOException { + if (out != null) { + out.close(); + } + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java (revision 0) @@ -0,0 +1,365 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; +import java.util.TreeMap; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocsProducer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; + +/** Handles a terms dict, but defers all details of postings + * reading to an instance of {@TermsDictDocsReader}. This + * terms dict codec is meant to be shared between + * different postings codecs, but, it's certainly possible + * to make a codec that has its own terms dict writer/reader. */ + +public class StandardTermsDictReader extends FieldsProducer { + private final IndexInput in; + + private final DocsProducer docs; + + final TreeMap fields = new TreeMap(); + + private final String segment; + private StandardTermsIndexReader indexReader; + + public StandardTermsDictReader(StandardTermsIndexReader indexReader, Directory dir, FieldInfos fieldInfos, String segment, DocsProducer docs, int readBufferSize) + throws IOException { + + in = dir.openInput(IndexFileNames.segmentFileName(segment, StandardCodec.TERMS_EXTENSION), readBufferSize); + this.segment = segment; + + boolean success = false; + try { + Codec.checkHeader(in, StandardTermsDictWriter.CODEC_NAME, StandardTermsDictWriter.VERSION_CURRENT); + + final long dirOffset = in.readLong(); + + this.docs = docs; + // Have DocsProducer init itself + docs.start(in); + + // Read per-field details + in.seek(dirOffset); + + final int numFields = in.readInt(); + + // mxx + if (Codec.DEBUG) { + System.out.println(Thread.currentThread().getName() + ": stdr create seg=" + segment + " numFields=" + numFields + " hasProx?=" + fieldInfos.hasProx()); + } + + for(int i=0;i= numTerms) { + return null; + } + if (Codec.DEBUG) { + System.out.println("tdr.next: field=" + fieldInfo.name + " termsInPointer=" + in.getFilePointer() + " vs len=" + in.length() + " seg=" + segment); + //new Throwable().printStackTrace(System.out); + } + bytesReader.read(); + docFreq = in.readVInt(); + if (Codec.DEBUG) { + System.out.println(" text=" + bytesReader.term + " freq=" + docFreq); + } + // TODO: would be cleaner, but space-wasting, to + // simply record a bit into each index entry as to + // whether it's an index entry or not... or, + // possibly store a "how many terms until next index + // entry" in each index entry, but that'd require + // some tricky lookahead work when writing the index + final boolean isIndex = indexReader.isIndexTerm(termUpto, docFreq); + + // mxx + // System.out.println(Thread.currentThread().getName() + ": isIndex=" + isIndex); + + docs.readTerm(docFreq, isIndex); + termUpto++; + if (Codec.DEBUG) { + System.out.println(" termUpto=" + termUpto + " vs numTerms=" + numTerms + " fp=" + in.getFilePointer()); + } + return bytesReader.term; + } + + public int docFreq() { + return docFreq; + } + + public DocsEnum docs(Bits skipDocs) throws IOException { + // nocommit + if (Codec.DEBUG) { + System.out.println("stdr.docs"); + } + DocsEnum docsEnum = docs.docs(skipDocs); + if (Codec.DEBUG) { + docsEnum.desc = fieldInfo.name + ":" + bytesReader.term; + } + return docsEnum; + } + } + } +} Property changes on: src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java (revision 0) +++ src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java (revision 0) @@ -0,0 +1,221 @@ +package org.apache.lucene.index.codecs.standard; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocsConsumer; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.UnicodeUtil; + +/** + * Writes terms dict and interacts with docs/positions + * consumers to write the postings files. + * + * The [new] terms dict format is field-centric: each field + * has its own section in the file. Fields are written in + * UTF16 string comparison order. Within each field, each + * term's text is written in UTF16 string comparison order. + */ + +public class StandardTermsDictWriter extends FieldsConsumer { + + final static String CODEC_NAME = "STANDARD_TERMS_DICT"; + + // Initial format + public static final int VERSION_START = 0; + + public static final int VERSION_CURRENT = VERSION_START; + + private final DeltaBytesWriter termWriter; + + final IndexOutput out; + final DocsConsumer consumer; + final FieldInfos fieldInfos; + FieldInfo currentField; + private final StandardTermsIndexWriter indexWriter; + private final List fields = new ArrayList(); + + // nocommit + private String segment; + + public StandardTermsDictWriter(StandardTermsIndexWriter indexWriter, SegmentWriteState state, DocsConsumer consumer) throws IOException { + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.TERMS_EXTENSION); + this.indexWriter = indexWriter; + out = state.directory.createOutput(termsFileName); + indexWriter.setTermsOutput(out); + state.flushedFiles.add(termsFileName); + this.segment = state.segmentName; + + if (Codec.DEBUG) { + System.out.println("stdw: write to segment=" + state.segmentName); + } + + fieldInfos = state.fieldInfos; + + // Count indexed fields up front + final int numFields = fieldInfos.size(); + Codec.writeHeader(out, CODEC_NAME, VERSION_CURRENT); + + out.writeLong(0); // leave space for end index pointer + + termWriter = new DeltaBytesWriter(out); + currentField = null; + this.consumer = consumer; + + consumer.start(out); // have consumer write its format/header + } + + public TermsConsumer addField(FieldInfo field) { + if (Codec.DEBUG) { + System.out.println("stdw.addField: field=" + field.name); + } + assert currentField == null || currentField.name.compareTo(field.name) < 0; + currentField = field; + StandardTermsIndexWriter.FieldWriter fieldIndexWriter = indexWriter.addField(field); + TermsConsumer terms = new TermsWriter(fieldIndexWriter, field, consumer); + fields.add(terms); + return terms; + } + + public void close() throws IOException { + + if (Codec.DEBUG) + System.out.println("stdw.close seg=" + segment); + + try { + final int fieldCount = fields.size(); + + if (Codec.DEBUG) + System.out.println(" numFields=" + fieldCount); + + final long dirStart = out.getFilePointer(); + + out.writeInt(fieldCount); + for(int i=0;i= 0; return sum * coordinator.coordFactors[coordinator.nrMatchers]; } Index: src/java/org/apache/lucene/search/ExactPhraseScorer.java =================================================================== --- src/java/org/apache/lucene/search/ExactPhraseScorer.java (revision 822088) +++ src/java/org/apache/lucene/search/ExactPhraseScorer.java (working copy) @@ -22,9 +22,9 @@ final class ExactPhraseScorer extends PhraseScorer { - ExactPhraseScorer(Weight weight, TermPositions[] tps, int[] offsets, + ExactPhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets, Similarity similarity, byte[] norms) { - super(weight, tps, offsets, similarity, norms); + super(weight, docs, offsets, similarity, norms); } protected final float phraseFreq() throws IOException { Index: src/java/org/apache/lucene/search/FieldCache.java =================================================================== --- src/java/org/apache/lucene/search/FieldCache.java (revision 822088) +++ src/java/org/apache/lucene/search/FieldCache.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.index.TermRef; import org.apache.lucene.document.NumericField; // for javadocs import org.apache.lucene.analysis.NumericTokenStream; // for javadocs @@ -101,7 +102,7 @@ */ public interface ByteParser extends Parser { /** Return a single Byte representation of this field's value. */ - public byte parseByte(String string); + public byte parseByte(TermRef term); } /** Interface to parse shorts from document fields. @@ -109,7 +110,7 @@ */ public interface ShortParser extends Parser { /** Return a short representation of this field's value. */ - public short parseShort(String string); + public short parseShort(TermRef term); } /** Interface to parse ints from document fields. @@ -117,7 +118,7 @@ */ public interface IntParser extends Parser { /** Return an integer representation of this field's value. */ - public int parseInt(String string); + public int parseInt(TermRef term); } /** Interface to parse floats from document fields. @@ -125,7 +126,7 @@ */ public interface FloatParser extends Parser { /** Return an float representation of this field's value. */ - public float parseFloat(String string); + public float parseFloat(TermRef term); } /** Interface to parse long from document fields. @@ -133,7 +134,7 @@ */ public interface LongParser extends Parser { /** Return an long representation of this field's value. */ - public long parseLong(String string); + public long parseLong(TermRef term); } /** Interface to parse doubles from document fields. @@ -141,16 +142,21 @@ */ public interface DoubleParser extends Parser { /** Return an long representation of this field's value. */ - public double parseDouble(String string); + public double parseDouble(TermRef term); } /** Expert: The cache used internally by sorting and range query classes. */ public static FieldCache DEFAULT = new FieldCacheImpl(); - + /** The default parser for byte values, which are encoded by {@link Byte#toString(byte)} */ public static final ByteParser DEFAULT_BYTE_PARSER = new ByteParser() { - public byte parseByte(String value) { - return Byte.parseByte(value); + public byte parseByte(TermRef term) { + final long num = FieldCacheImpl.parseLong(term); + if (num >= Byte.MIN_VALUE && num <= Byte.MAX_VALUE) { + return (byte) num; + } else { + throw new IllegalArgumentException("value \"" + term + "\" is out of bounds for Byte"); + } } protected Object readResolve() { return DEFAULT_BYTE_PARSER; @@ -162,8 +168,13 @@ /** The default parser for short values, which are encoded by {@link Short#toString(short)} */ public static final ShortParser DEFAULT_SHORT_PARSER = new ShortParser() { - public short parseShort(String value) { - return Short.parseShort(value); + public short parseShort(TermRef term) { + final long num = FieldCacheImpl.parseLong(term); + if (num >= Short.MIN_VALUE && num <= Short.MAX_VALUE) { + return (short) num; + } else { + throw new IllegalArgumentException("value \"" + term + "\" is out of bounds for Short"); + } } protected Object readResolve() { return DEFAULT_SHORT_PARSER; @@ -175,8 +186,13 @@ /** The default parser for int values, which are encoded by {@link Integer#toString(int)} */ public static final IntParser DEFAULT_INT_PARSER = new IntParser() { - public int parseInt(String value) { - return Integer.parseInt(value); + public int parseInt(TermRef term) { + final long num = FieldCacheImpl.parseLong(term); + if (num >= Integer.MIN_VALUE && num <= Integer.MAX_VALUE) { + return (int) num; + } else { + throw new IllegalArgumentException("value \"" + term + "\" is out of bounds for Int"); + } } protected Object readResolve() { return DEFAULT_INT_PARSER; @@ -188,8 +204,10 @@ /** The default parser for float values, which are encoded by {@link Float#toString(float)} */ public static final FloatParser DEFAULT_FLOAT_PARSER = new FloatParser() { - public float parseFloat(String value) { - return Float.parseFloat(value); + public float parseFloat(TermRef term) { + // TODO: would be far better to directly parse + // the UTF-8 bytes into float, but that's tricky? + return Float.parseFloat(term.toString()); } protected Object readResolve() { return DEFAULT_FLOAT_PARSER; @@ -201,8 +219,8 @@ /** The default parser for long values, which are encoded by {@link Long#toString(long)} */ public static final LongParser DEFAULT_LONG_PARSER = new LongParser() { - public long parseLong(String value) { - return Long.parseLong(value); + public long parseLong(TermRef term) { + return FieldCacheImpl.parseLong(term); } protected Object readResolve() { return DEFAULT_LONG_PARSER; @@ -214,8 +232,10 @@ /** The default parser for double values, which are encoded by {@link Double#toString(double)} */ public static final DoubleParser DEFAULT_DOUBLE_PARSER = new DoubleParser() { - public double parseDouble(String value) { - return Double.parseDouble(value); + public double parseDouble(TermRef term) { + // TODO: would be far better to directly parse + // the UTF-8 bytes into float, but that's tricky? + return Double.parseDouble(term.toString()); } protected Object readResolve() { return DEFAULT_DOUBLE_PARSER; @@ -230,8 +250,8 @@ * via {@link NumericField}/{@link NumericTokenStream}. */ public static final IntParser NUMERIC_UTILS_INT_PARSER=new IntParser(){ - public int parseInt(String val) { - final int shift = val.charAt(0)-NumericUtils.SHIFT_START_INT; + public int parseInt(TermRef val) { + final int shift = val.bytes[val.offset]-NumericUtils.SHIFT_START_INT; if (shift>0 && shift<=31) throw new FieldCacheImpl.StopFillCacheException(); return NumericUtils.prefixCodedToInt(val); @@ -249,11 +269,11 @@ * via {@link NumericField}/{@link NumericTokenStream}. */ public static final FloatParser NUMERIC_UTILS_FLOAT_PARSER=new FloatParser(){ - public float parseFloat(String val) { - final int shift = val.charAt(0)-NumericUtils.SHIFT_START_INT; + public float parseFloat(TermRef term) { + final int shift = term.bytes[term.offset]-NumericUtils.SHIFT_START_INT; if (shift>0 && shift<=31) throw new FieldCacheImpl.StopFillCacheException(); - return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(val)); + return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term)); } protected Object readResolve() { return NUMERIC_UTILS_FLOAT_PARSER; @@ -268,11 +288,11 @@ * via {@link NumericField}/{@link NumericTokenStream}. */ public static final LongParser NUMERIC_UTILS_LONG_PARSER = new LongParser(){ - public long parseLong(String val) { - final int shift = val.charAt(0)-NumericUtils.SHIFT_START_LONG; + public long parseLong(TermRef term) { + final int shift = term.bytes[term.offset]-NumericUtils.SHIFT_START_LONG; if (shift>0 && shift<=63) throw new FieldCacheImpl.StopFillCacheException(); - return NumericUtils.prefixCodedToLong(val); + return NumericUtils.prefixCodedToLong(term); } protected Object readResolve() { return NUMERIC_UTILS_LONG_PARSER; @@ -287,11 +307,11 @@ * via {@link NumericField}/{@link NumericTokenStream}. */ public static final DoubleParser NUMERIC_UTILS_DOUBLE_PARSER = new DoubleParser(){ - public double parseDouble(String val) { - final int shift = val.charAt(0)-NumericUtils.SHIFT_START_LONG; + public double parseDouble(TermRef term) { + final int shift = term.bytes[term.offset]-NumericUtils.SHIFT_START_LONG; if (shift>0 && shift<=63) throw new FieldCacheImpl.StopFillCacheException(); - return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(val)); + return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term)); } protected Object readResolve() { return NUMERIC_UTILS_DOUBLE_PARSER; Index: src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 822088) +++ src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy) @@ -29,9 +29,14 @@ import org.apache.lucene.document.NumericField; // javadoc import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermDocs; // deprecated +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.FieldCacheSanityChecker; @@ -336,22 +341,28 @@ return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER); } final byte[] retArray = new byte[reader.maxDoc()]; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term (field)); - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; - byte termval = parser.parseByte(term.text()); - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = termval; + Terms terms = reader.fields().terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + try { + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; + } + final byte termval = parser.parseByte(term); + final DocsEnum docs = termsEnum.docs(delDocs); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = termval; + } } - } while (termEnum.next()); - } catch (StopFillCacheException stop) { - } finally { - termDocs.close(); - termEnum.close(); + } catch (StopFillCacheException stop) { + } } return retArray; } @@ -382,22 +393,28 @@ return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER); } final short[] retArray = new short[reader.maxDoc()]; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term (field)); - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; - short termval = parser.parseShort(term.text()); - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = termval; + Terms terms = reader.fields().terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + try { + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; + } + final short termval = parser.parseShort(term); + final DocsEnum docs = termsEnum.docs(delDocs); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = termval; + } } - } while (termEnum.next()); - } catch (StopFillCacheException stop) { - } finally { - termDocs.close(); - termEnum.close(); + } catch (StopFillCacheException stop) { + } } return retArray; } @@ -432,27 +449,40 @@ } } int[] retArray = null; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term (field)); - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; - int termval = parser.parseInt(term.text()); - if (retArray == null) // late init - retArray = new int[reader.maxDoc()]; - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = termval; + + Terms terms = reader.fields().terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + try { + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; + } + final int termval = parser.parseInt(term); + if (retArray == null) { + // late init so numeric fields don't double allocate + retArray = new int[reader.maxDoc()]; + } + + final DocsEnum docs = termsEnum.docs(delDocs); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = termval; + } } - } while (termEnum.next()); - } catch (StopFillCacheException stop) { - } finally { - termDocs.close(); - termEnum.close(); + } catch (StopFillCacheException stop) { + } } - if (retArray == null) // no values + + if (retArray == null) { + // no values retArray = new int[reader.maxDoc()]; + } return retArray; } }; @@ -487,29 +517,42 @@ } catch (NumberFormatException ne) { return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER); } - } + } float[] retArray = null; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term (field)); - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; - float termval = parser.parseFloat(term.text()); - if (retArray == null) // late init - retArray = new float[reader.maxDoc()]; - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = termval; + + Terms terms = reader.fields().terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + try { + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; + } + final float termval = parser.parseFloat(term); + if (retArray == null) { + // late init so numeric fields don't double allocate + retArray = new float[reader.maxDoc()]; + } + + final DocsEnum docs = termsEnum.docs(delDocs); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = termval; + } } - } while (termEnum.next()); - } catch (StopFillCacheException stop) { - } finally { - termDocs.close(); - termEnum.close(); + } catch (StopFillCacheException stop) { + } } - if (retArray == null) // no values + + if (retArray == null) { + // no values retArray = new float[reader.maxDoc()]; + } return retArray; } }; @@ -549,27 +592,39 @@ } } long[] retArray = null; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term(field)); - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; - long termval = parser.parseLong(term.text()); - if (retArray == null) // late init - retArray = new long[reader.maxDoc()]; - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = termval; + Terms terms = reader.fields().terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + try { + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; + } + final long termval = parser.parseLong(term); + if (retArray == null) { + // late init so numeric fields don't double allocate + retArray = new long[reader.maxDoc()]; + } + + final DocsEnum docs = termsEnum.docs(delDocs); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = termval; + } } - } while (termEnum.next()); - } catch (StopFillCacheException stop) { - } finally { - termDocs.close(); - termEnum.close(); + } catch (StopFillCacheException stop) { + } } - if (retArray == null) // no values + + if (retArray == null) { + // no values retArray = new long[reader.maxDoc()]; + } return retArray; } }; @@ -610,24 +665,33 @@ } } double[] retArray = null; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term (field)); - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; - double termval = parser.parseDouble(term.text()); - if (retArray == null) // late init - retArray = new double[reader.maxDoc()]; - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = termval; + Terms terms = reader.fields().terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + try { + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; + } + final double termval = parser.parseDouble(term); + if (retArray == null) { + // late init so numeric fields don't double allocate + retArray = new double[reader.maxDoc()]; + } + + final DocsEnum docs = termsEnum.docs(delDocs); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = termval; + } } - } while (termEnum.next()); - } catch (StopFillCacheException stop) { - } finally { - termDocs.close(); - termEnum.close(); + } catch (StopFillCacheException stop) { + } } if (retArray == null) // no values retArray = new double[reader.maxDoc()]; @@ -650,21 +714,26 @@ throws IOException { String field = StringHelper.intern((String) entryKey.field); final String[] retArray = new String[reader.maxDoc()]; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term (field)); - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; - String termval = term.text(); - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = termval; + + Terms terms = reader.fields().terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; } - } while (termEnum.next()); - } finally { - termDocs.close(); - termEnum.close(); + final DocsEnum docs = termsEnum.docs(delDocs); + final String termval = term.toString(); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = termval; + } + } } return retArray; } @@ -686,8 +755,9 @@ String field = StringHelper.intern((String) entryKey.field); final int[] retArray = new int[reader.maxDoc()]; String[] mterms = new String[reader.maxDoc()+1]; - TermDocs termDocs = reader.termDocs(); - TermEnum termEnum = reader.terms (new Term (field)); + + Terms terms = reader.fields().terms(field); + int t = 0; // current term number // an entry for documents that have no terms in this field @@ -696,28 +766,34 @@ // needs to change as well. mterms[t++] = null; - try { - do { - Term term = termEnum.term(); - if (term==null || term.field() != field) break; + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(); + final Bits delDocs = reader.getDeletedDocs(); + while(true) { + final TermRef term = termsEnum.next(); + if (term == null) { + break; + } // store term text // we expect that there is at most one term per document - if (t >= mterms.length) throw new RuntimeException ("there are more terms than " + - "documents in field \"" + field + "\", but it's impossible to sort on " + - "tokenized fields"); - mterms[t] = term.text(); - - termDocs.seek (termEnum); - while (termDocs.next()) { - retArray[termDocs.doc()] = t; + if (t >= mterms.length) { + throw new RuntimeException ("there are more terms than " + + "documents in field \"" + field + "\", but it's impossible to sort on " + + "tokenized fields"); } + mterms[t] = term.toString(); + final DocsEnum docs = termsEnum.docs(delDocs); + while (true) { + final int docID = docs.next(); + if (docID == DocsEnum.NO_MORE_DOCS) { + break; + } + retArray[docID] = t; + } t++; - } while (termEnum.next()); - } finally { - termDocs.close(); - termEnum.close(); + } } if (t == 0) { @@ -727,9 +803,9 @@ } else if (t < mterms.length) { // if there are less terms than documents, // trim off the dead array space - String[] terms = new String[t]; - System.arraycopy (mterms, 0, terms, 0, t); - mterms = terms; + String[] newTerms = new String[t]; + System.arraycopy (mterms, 0, newTerms, 0, t); + mterms = newTerms; } StringIndex value = new StringIndex (retArray, mterms); @@ -820,7 +896,7 @@ String field = entry.field; SortComparator comparator = (SortComparator) entry.custom; final Comparable[] retArray = new Comparable[reader.maxDoc()]; - TermDocs termDocs = reader.termDocs(); + TermDocs termDocs = reader.termDocs(); // deprecated TermEnum termEnum = reader.terms (new Term (field)); try { do { @@ -849,5 +925,29 @@ public PrintStream getInfoStream() { return infoStream; } + + // Directly parses a numeric value from UTF8 bytes + // nocommit -- whitespace? +e syntax? + final static long parseLong(TermRef term) { + int upto = term.offset; + final int negMul; + if (term.bytes[upto] == '-') { + negMul = -1; + upto++; + } else { + negMul = 1; + } + final int end = term.offset + term.length; + long number = 0; + while(upto < end) { + final int b = term.bytes[upto++]; + if (b >= '0' && b <= '9') { + number = 10*number + (int) (b-'0'); + } else { + throw new NumberFormatException("could not parse \"" + term + "\" to a number"); + } + } + return negMul * number; + } } Index: src/java/org/apache/lucene/search/FieldCacheRangeFilter.java =================================================================== --- src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (revision 822088) +++ src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (working copy) @@ -19,8 +19,8 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.TermDocs; import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.Bits; import org.apache.lucene.document.NumericField; // for javadocs /** @@ -117,9 +117,9 @@ assert inclusiveLowerPoint > 0 && inclusiveUpperPoint > 0; - // for this DocIdSet, we never need to use TermDocs, + // for this DocIdSet, we can ignore deleted docs // because deleted docs have an order of 0 (null entry in StringIndex) - return new FieldCacheDocIdSet(reader, false) { + return new FieldCacheDocIdSet(reader, true) { final boolean matchDoc(int doc) { return fcsi.order[doc] >= inclusiveLowerPoint && fcsi.order[doc] <= inclusiveUpperPoint; } @@ -167,8 +167,8 @@ return DocIdSet.EMPTY_DOCIDSET; final byte[] values = FieldCache.DEFAULT.getBytes(reader, field, (FieldCache.ByteParser) parser); - // we only request the usage of termDocs, if the range contains 0 - return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) { + // we only respect deleted docs if the range contains 0 + return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) { boolean matchDoc(int doc) { return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint; } @@ -216,8 +216,8 @@ return DocIdSet.EMPTY_DOCIDSET; final short[] values = FieldCache.DEFAULT.getShorts(reader, field, (FieldCache.ShortParser) parser); - // we only request the usage of termDocs, if the range contains 0 - return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) { + // ignore deleted docs if range doesn't contain 0 + return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) { boolean matchDoc(int doc) { return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint; } @@ -265,8 +265,8 @@ return DocIdSet.EMPTY_DOCIDSET; final int[] values = FieldCache.DEFAULT.getInts(reader, field, (FieldCache.IntParser) parser); - // we only request the usage of termDocs, if the range contains 0 - return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) { + // ignore deleted docs if range doesn't contain 0 + return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) { boolean matchDoc(int doc) { return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint; } @@ -314,8 +314,8 @@ return DocIdSet.EMPTY_DOCIDSET; final long[] values = FieldCache.DEFAULT.getLongs(reader, field, (FieldCache.LongParser) parser); - // we only request the usage of termDocs, if the range contains 0 - return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0L && inclusiveUpperPoint >= 0L)) { + // ignore deleted docs if range doesn't contain 0 + return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0L && inclusiveUpperPoint >= 0L)) { boolean matchDoc(int doc) { return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint; } @@ -367,8 +367,8 @@ return DocIdSet.EMPTY_DOCIDSET; final float[] values = FieldCache.DEFAULT.getFloats(reader, field, (FieldCache.FloatParser) parser); - // we only request the usage of termDocs, if the range contains 0 - return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0.0f && inclusiveUpperPoint >= 0.0f)) { + // ignore deleted docs if range doesn't contain 0 + return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0.0f && inclusiveUpperPoint >= 0.0f)) { boolean matchDoc(int doc) { return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint; } @@ -420,8 +420,8 @@ return DocIdSet.EMPTY_DOCIDSET; final double[] values = FieldCache.DEFAULT.getDoubles(reader, field, (FieldCache.DoubleParser) parser); - // we only request the usage of termDocs, if the range contains 0 - return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0.0 && inclusiveUpperPoint >= 0.0)) { + // ignore deleted docs if range doesn't contain 0 + return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0.0 && inclusiveUpperPoint >= 0.0)) { boolean matchDoc(int doc) { return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint; } @@ -467,19 +467,20 @@ static abstract class FieldCacheDocIdSet extends DocIdSet { private final IndexReader reader; - private boolean mayUseTermDocs; + private boolean canIgnoreDeletedDocs; - FieldCacheDocIdSet(IndexReader reader, boolean mayUseTermDocs) { + FieldCacheDocIdSet(IndexReader reader, boolean canIgnoreDeletedDocs) { this.reader = reader; - this.mayUseTermDocs = mayUseTermDocs; + this.canIgnoreDeletedDocs = canIgnoreDeletedDocs; } /** this method checks, if a doc is a hit, should throw AIOBE, when position invalid */ abstract boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException; - /** this DocIdSet is cacheable, if it works solely with FieldCache and no TermDocs */ + /** this DocIdSet is cacheable, if it can ignore + * deletions */ public boolean isCacheable() { - return !(mayUseTermDocs && reader.hasDeletions()); + return canIgnoreDeletedDocs || !reader.hasDeletions(); } public DocIdSetIterator iterator() throws IOException { @@ -487,101 +488,64 @@ // can change after call to hasDeletions until TermDocs creation. // We only use an iterator with termDocs, when this was requested (e.g. range contains 0) // and the index has deletions - final TermDocs termDocs; + + final Bits skipDocs; synchronized(reader) { - termDocs = isCacheable() ? null : reader.termDocs(null); + if (isCacheable()) { + skipDocs = null; + } else { + skipDocs = reader.getDeletedDocs(); + } } - if (termDocs != null) { - // a DocIdSetIterator using TermDocs to iterate valid docIds - return new DocIdSetIterator() { - private int doc = -1; + final int maxDoc = reader.maxDoc(); + + // a DocIdSetIterator generating docIds by + // incrementing a variable & checking skipDocs - + return new DocIdSetIterator() { + private int doc = -1; - /** @deprecated use {@link #nextDoc()} instead. */ - public boolean next() throws IOException { - return nextDoc() != NO_MORE_DOCS; - } + /** @deprecated use {@link #nextDoc()} instead. */ + public boolean next() throws IOException { + return nextDoc() != NO_MORE_DOCS; + } - /** @deprecated use {@link #advance(int)} instead. */ - public boolean skipTo(int target) throws IOException { - return advance(target) != NO_MORE_DOCS; - } + /** @deprecated use {@link #advance(int)} instead. */ + public boolean skipTo(int target) throws IOException { + return advance(target) != NO_MORE_DOCS; + } - /** @deprecated use {@link #docID()} instead. */ - public int doc() { - return termDocs.doc(); - } - - public int docID() { - return doc; - } + /** @deprecated use {@link #docID()} instead. */ + public int doc() { + return doc; + } + + public int docID() { + return doc; + } - public int nextDoc() throws IOException { + public int nextDoc() { + try { do { - if (!termDocs.next()) - return doc = NO_MORE_DOCS; - } while (!matchDoc(doc = termDocs.doc())); + doc++; + } while ((skipDocs != null && doc < maxDoc && skipDocs.get(doc)) || !matchDoc(doc)); return doc; + } catch (ArrayIndexOutOfBoundsException e) { + return doc = NO_MORE_DOCS; } + } - public int advance(int target) throws IOException { - if (!termDocs.skipTo(target)) - return doc = NO_MORE_DOCS; - while (!matchDoc(doc = termDocs.doc())) { - if (!termDocs.next()) - return doc = NO_MORE_DOCS; + public int advance(int target) { + try { + doc = target; + while (!matchDoc(doc)) { + doc++; } return doc; - } - }; - } else { - // a DocIdSetIterator generating docIds by incrementing a variable - - // this one can be used if there are no deletions are on the index - return new DocIdSetIterator() { - private int doc = -1; - - /** @deprecated use {@link #nextDoc()} instead. */ - public boolean next() throws IOException { - return nextDoc() != NO_MORE_DOCS; + } catch (ArrayIndexOutOfBoundsException e) { + return doc = NO_MORE_DOCS; } - - /** @deprecated use {@link #advance(int)} instead. */ - public boolean skipTo(int target) throws IOException { - return advance(target) != NO_MORE_DOCS; - } - - /** @deprecated use {@link #docID()} instead. */ - public int doc() { - return doc; - } - - public int docID() { - return doc; - } - - public int nextDoc() { - try { - do { - doc++; - } while (!matchDoc(doc)); - return doc; - } catch (ArrayIndexOutOfBoundsException e) { - return doc = NO_MORE_DOCS; - } - } - - public int advance(int target) { - try { - doc = target; - while (!matchDoc(doc)) { - doc++; - } - return doc; - } catch (ArrayIndexOutOfBoundsException e) { - return doc = NO_MORE_DOCS; - } - } - }; - } + } + }; } } Index: src/java/org/apache/lucene/search/FilteredTermEnum.java =================================================================== --- src/java/org/apache/lucene/search/FilteredTermEnum.java (revision 822088) +++ src/java/org/apache/lucene/search/FilteredTermEnum.java (working copy) @@ -24,7 +24,10 @@ /** Abstract class for enumerating a subset of all terms.

Term enumerations are always ordered by Term.compareTo(). Each term in - the enumeration is greater than all that precede it. */ + the enumeration is greater than all that precede it. + + @deprecated Switch to {@link FilteredTermsEnum} instead. +*/ public abstract class FilteredTermEnum extends TermEnum { /** the current term */ protected Term currentTerm = null; Index: src/java/org/apache/lucene/search/FilteredTermsEnum.java =================================================================== --- src/java/org/apache/lucene/search/FilteredTermsEnum.java (revision 0) +++ src/java/org/apache/lucene/search/FilteredTermsEnum.java (revision 0) @@ -0,0 +1,155 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.Bits; + +/** + * Abstract class for enumerating a subset of all terms. + * + *

On creation, the enumerator must already be position + * to the first term.

+ * + *

Term enumerations are always ordered by + * Term.compareTo(). Each term in the enumeration is + * greater than all that precede it.

+*/ +public abstract class FilteredTermsEnum extends TermsEnum { + + /** the delegate enum - to set this member use {@link #init} */ + protected TermsEnum actualEnum; + + /** Return true if term is acceptd */ + protected abstract boolean accept(TermRef term); + + /** Equality measure on the term */ + public abstract float difference(); + + /** Return true if the enumeration should now stop */ + protected abstract boolean end(); + + boolean done; + + protected abstract String field(); + + protected TermRef current; + + /** + * use this method to set the actual TermsEnum (e.g. in ctor), + * it will be automatically positioned on the first matching term. + */ + protected void init(TermsEnum actualEnum, TermRef term) throws IOException { + this.actualEnum = actualEnum; + done = false; + + // Find the first term that matches + if (term != null) { + SeekResult result = actualEnum.seek(term); + + final TermRef foundTerm; + if (result.status == SeekResult.Status.FOUND) { + foundTerm = term; + } else if (result.status == SeekResult.Status.NOT_FOUND) { + foundTerm = result.term; + } else { + foundTerm = null; + done = true; + } + if (foundTerm != null) { + if (accept(foundTerm)) { + current = foundTerm; + } else { + next(); + } + } + } + } + + // nocommit -- do we really need this? caller can keep + // track of it? + protected TermRef term() { + return current; + } + + /** + * Returns the docFreq of the current Term in the enumeration. + * Returns -1 if no Term matches or all terms have been enumerated. + */ + public int docFreq() { + assert actualEnum != null; + return actualEnum.docFreq(); + } + + /** Increments the enumeration to the next element. True if one exists. */ + public TermRef next() throws IOException { + assert actualEnum != null; + while (true) { + if (done || end()) { + current = null; + return null; + } + TermRef term = actualEnum.next(); + if (term != null) { + if (accept(term)) { + current = term; + return current; + } + } else { + current = null; + return null; + } + } + } + + public SeekResult seek(TermRef term) throws IOException { + SeekResult result = actualEnum.seek(term); + if (result.status == SeekResult.Status.FOUND) { + if (!accept(term)) { + TermRef term2 = next(); + if (term2 == null) { + result.status = SeekResult.Status.END; + } else { + result.status = SeekResult.Status.NOT_FOUND; + result.term = term2; + } + } else { + current = term; + } + } else if (result.status == SeekResult.Status.NOT_FOUND) { + if (!accept(result.term)) { + TermRef term2 = next(); + if (term2 == null) { + result.status = SeekResult.Status.END; + } else { + result.term = term2; + } + } else { + current = term; + } + } + return result; + } + + public DocsEnum docs(Bits bits) throws IOException { + return actualEnum.docs(bits); + } +} Index: src/java/org/apache/lucene/search/FuzzyQuery.java =================================================================== --- src/java/org/apache/lucene/search/FuzzyQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/FuzzyQuery.java (working copy) @@ -114,6 +114,10 @@ return new FuzzyTermEnum(reader, getTerm(), minimumSimilarity, prefixLength); } + protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException { + return new FuzzyTermsEnum(reader, getTerm(), minimumSimilarity, prefixLength); + } + /** * Returns the pattern term. */ Index: src/java/org/apache/lucene/search/FuzzyTermEnum.java =================================================================== --- src/java/org/apache/lucene/search/FuzzyTermEnum.java (revision 822088) +++ src/java/org/apache/lucene/search/FuzzyTermEnum.java (working copy) @@ -27,6 +27,8 @@ * *

Term enumerations are always ordered by Term.compareTo(). Each term in * the enumeration is greater than all that precede it. + * + * @deprecated Please use {@link FuzzyTermsEnum} instead. */ public final class FuzzyTermEnum extends FilteredTermEnum { Index: src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 0) +++ src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 0) @@ -0,0 +1,317 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermRef; + +import java.io.IOException; + +/** Subclass of FilteredTermEnum for enumerating all terms that are similar + * to the specified filter term. + * + *

Term enumerations are always ordered by Term.compareTo(). Each term in + * the enumeration is greater than all that precede it. + */ +public final class FuzzyTermsEnum extends FilteredTermsEnum { + + /* This should be somewhere around the average long word. + * If it is longer, we waste time and space. If it is shorter, we waste a + * little bit of time growing the array as we encounter longer words. + */ + private static final int TYPICAL_LONGEST_WORD_IN_INDEX = 19; + + /* Allows us save time required to create a new array + * every time similarity is called. + */ + private int[][] d; + + private float similarity; + private boolean end; + + private Term searchTerm; + private final String field; + private final String text; + private final String prefix; + + private final float minimumSimilarity; + private final float scale_factor; + private final int[] maxDistances = new int[TYPICAL_LONGEST_WORD_IN_INDEX]; + + // nocommit -- remove some of these ctors: + /** + * Creates a FuzzyTermEnum with an empty prefix and a minSimilarity of 0.5f. + *

+ * After calling the constructor the enumeration is already pointing to the first + * valid term if such a term exists. + * + * @param reader + * @param term + * @throws IOException + * @see #FuzzyTermEnum(IndexReader, Term, float, int) + */ + public FuzzyTermsEnum(IndexReader reader, Term term) throws IOException { + this(reader, term, FuzzyQuery.defaultMinSimilarity, FuzzyQuery.defaultPrefixLength); + } + + /** + * Creates a FuzzyTermEnum with an empty prefix. + *

+ * After calling the constructor the enumeration is already pointing to the first + * valid term if such a term exists. + * + * @param reader + * @param term + * @param minSimilarity + * @throws IOException + * @see #FuzzyTermEnum(IndexReader, Term, float, int) + */ + public FuzzyTermsEnum(IndexReader reader, Term term, float minSimilarity) throws IOException { + this(reader, term, minSimilarity, FuzzyQuery.defaultPrefixLength); + } + + /** + * Constructor for enumeration of all terms from specified reader which share a prefix of + * length prefixLength with term and which have a fuzzy similarity > + * minSimilarity. + *

+ * After calling the constructor the enumeration is already pointing to the first + * valid term if such a term exists. + * + * @param reader Delivers terms. + * @param term Pattern term. + * @param minSimilarity Minimum required similarity for terms from the reader. Default value is 0.5f. + * @param prefixLength Length of required common prefix. Default value is 0. + * @throws IOException + */ + public FuzzyTermsEnum(IndexReader reader, Term term, final float minSimilarity, final int prefixLength) throws IOException { + super(); + + if (minSimilarity >= 1.0f) + throw new IllegalArgumentException("minimumSimilarity cannot be greater than or equal to 1"); + else if (minSimilarity < 0.0f) + throw new IllegalArgumentException("minimumSimilarity cannot be less than 0"); + if(prefixLength < 0) + throw new IllegalArgumentException("prefixLength cannot be less than 0"); + + this.minimumSimilarity = minSimilarity; + this.scale_factor = 1.0f / (1.0f - minimumSimilarity); + this.searchTerm = term; + this.field = searchTerm.field(); + + //The prefix could be longer than the word. + //It's kind of silly though. It means we must match the entire word. + final int fullSearchTermLength = searchTerm.text().length(); + final int realPrefixLength = prefixLength > fullSearchTermLength ? fullSearchTermLength : prefixLength; + + this.text = searchTerm.text().substring(realPrefixLength); + this.prefix = searchTerm.text().substring(0, realPrefixLength); + prefixTermRef = new TermRef(prefix); + initializeMaxDistances(); + this.d = initDistanceArray(); + + Terms terms = reader.fields().terms(field); + if (terms != null) { + init(terms.iterator(), prefixTermRef); + } else { + end = true; + } + } + + private final TermRef prefixTermRef; + + protected String field() { + return field; + } + + /** + * The termCompare method in FuzzyTermEnum uses Levenshtein distance to + * calculate the distance between the given term and the comparing term. + */ + protected final boolean accept(TermRef term) { + if (term.startsWith(prefixTermRef)) { + // TODO: costly that we create intermediate String: + final String target = term.toString().substring(prefix.length()); + this.similarity = similarity(target); + return (similarity > minimumSimilarity); + } + end = true; + return false; + } + + public final float difference() { + return (float)((similarity - minimumSimilarity) * scale_factor); + } + + public final boolean end() { + return end; + } + + /****************************** + * Compute Levenshtein distance + ******************************/ + + /** + * Finds and returns the smallest of three integers + */ + private static final int min(int a, int b, int c) { + final int t = (a < b) ? a : b; + return (t < c) ? t : c; + } + + private final int[][] initDistanceArray(){ + return new int[this.text.length() + 1][TYPICAL_LONGEST_WORD_IN_INDEX]; + } + + /** + *

Similarity returns a number that is 1.0f or less (including negative numbers) + * based on how similar the Term is compared to a target term. It returns + * exactly 0.0f when + * 

+   *    editDistance < maximumEditDistance
+ * Otherwise it returns: + * 
+   *    1 - (editDistance / length)
+ * where length is the length of the shortest term (text or target) including a + * prefix that are identical and editDistance is the Levenshtein distance for + * the two words.

+ * + *

Embedded within this algorithm is a fail-fast Levenshtein distance + * algorithm. The fail-fast algorithm differs from the standard Levenshtein + * distance algorithm in that it is aborted if it is discovered that the + * minimum distance between the words is greater than some threshold. + * + *

To calculate the maximum distance threshold we use the following formula: + * 

+   *     (1 - minimumSimilarity) * length
+ * where length is the shortest term including any prefix that is not part of the + * similarity comparison. This formula was derived by solving for what maximum value + * of distance returns false for the following statements: + * 
+   *   similarity = 1 - ((float)distance / (float) (prefixLength + Math.min(textlen, targetlen)));
+   *   return (similarity > minimumSimilarity);
+ * where distance is the Levenshtein distance for the two words. + *

+ *

Levenshtein distance (also known as edit distance) is a measure of similarity + * between two strings where the distance is measured as the number of character + * deletions, insertions or substitutions required to transform one string to + * the other string. + * @param target the target word or phrase + * @return the similarity, 0.0 or less indicates that it matches less than the required + * threshold and 1.0 indicates that the text and target are identical + */ + private synchronized final float similarity(final String target) { + final int m = target.length(); + final int n = text.length(); + if (n == 0) { + //we don't have anything to compare. That means if we just add + //the letters for m we get the new word + return prefix.length() == 0 ? 0.0f : 1.0f - ((float) m / prefix.length()); + } + if (m == 0) { + return prefix.length() == 0 ? 0.0f : 1.0f - ((float) n / prefix.length()); + } + + final int maxDistance = getMaxDistance(m); + + if (maxDistance < Math.abs(m-n)) { + //just adding the characters of m to n or vice-versa results in + //too many edits + //for example "pre" length is 3 and "prefixes" length is 8. We can see that + //given this optimal circumstance, the edit distance cannot be less than 5. + //which is 8-3 or more precisely Math.abs(3-8). + //if our maximum edit distance is 4, then we can discard this word + //without looking at it. + return 0.0f; + } + + //let's make sure we have enough room in our array to do the distance calculations. + if (d[0].length <= m) { + growDistanceArray(m); + } + + // init matrix d + for (int i = 0; i <= n; i++) d[i][0] = i; + for (int j = 0; j <= m; j++) d[0][j] = j; + + // start computing edit distance + for (int i = 1; i <= n; i++) { + int bestPossibleEditDistance = m; + final char s_i = text.charAt(i - 1); + for (int j = 1; j <= m; j++) { + if (s_i != target.charAt(j-1)) { + d[i][j] = min(d[i-1][j], d[i][j-1], d[i-1][j-1])+1; + } + else { + d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]); + } + bestPossibleEditDistance = Math.min(bestPossibleEditDistance, d[i][j]); + } + + //After calculating row i, the best possible edit distance + //can be found by found by finding the smallest value in a given column. + //If the bestPossibleEditDistance is greater than the max distance, abort. + + if (i > maxDistance && bestPossibleEditDistance > maxDistance) { //equal is okay, but not greater + //the closest the target can be to the text is just too far away. + //this target is leaving the party early. + return 0.0f; + } + } + + // this will return less than 0.0 when the edit distance is + // greater than the number of characters in the shorter word. + // but this was the formula that was previously used in FuzzyTermEnum, + // so it has not been changed (even though minimumSimilarity must be + // greater than 0.0) + return 1.0f - ((float)d[n][m] / (float) (prefix.length() + Math.min(n, m))); + } + + /** + * Grow the second dimension of the array, so that we can calculate the + * Levenshtein difference. + */ + private void growDistanceArray(int m) { + for (int i = 0; i < d.length; i++) { + d[i] = new int[m+1]; + } + } + + /** + * The max Distance is the maximum Levenshtein distance for the text + * compared to some other value that results in score that is + * better than the minimum similarity. + * @param m the length of the "other value" + * @return the maximum levenshtein distance that we care about + */ + private final int getMaxDistance(int m) { + return (m < maxDistances.length) ? maxDistances[m] : calculateMaxDistance(m); + } + + private void initializeMaxDistances() { + for (int i = 0; i < maxDistances.length; i++) { + maxDistances[i] = calculateMaxDistance(i); + } + } + + private int calculateMaxDistance(int m) { + return (int) ((1-minimumSimilarity) * (Math.min(text.length(), m) + prefix.length())); + } +} Index: src/java/org/apache/lucene/search/MatchAllDocsQuery.java =================================================================== --- src/java/org/apache/lucene/search/MatchAllDocsQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/MatchAllDocsQuery.java (working copy) @@ -18,8 +18,8 @@ */ import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.TermDocs; import org.apache.lucene.util.ToStringUtils; +import org.apache.lucene.util.Bits; import java.util.Set; import java.io.IOException; @@ -44,16 +44,18 @@ } private class MatchAllScorer extends Scorer { - final TermDocs termDocs; final float score; final byte[] norms; private int doc = -1; + private final int maxDoc; + private final Bits delDocs; MatchAllScorer(IndexReader reader, Similarity similarity, Weight w, byte[] norms) throws IOException { super(similarity); - this.termDocs = reader.termDocs(null); + delDocs = reader.getDeletedDocs(); score = w.getValue(); + maxDoc = reader.maxDoc(); this.norms = norms; } @@ -63,7 +65,7 @@ /** @deprecated use {@link #docID()} instead. */ public int doc() { - return termDocs.doc(); + return doc; } public int docID() { @@ -76,7 +78,14 @@ } public int nextDoc() throws IOException { - return doc = termDocs.next() ? termDocs.doc() : NO_MORE_DOCS; + doc++; + while(delDocs != null && doc < maxDoc && delDocs.get(doc)) { + doc++; + } + if (doc == maxDoc) { + doc = NO_MORE_DOCS; + } + return doc; } public float score() { @@ -89,7 +98,8 @@ } public int advance(int target) throws IOException { - return doc = termDocs.skipTo(target) ? termDocs.doc() : NO_MORE_DOCS; + doc = target-1; + return nextDoc(); } } Index: src/java/org/apache/lucene/search/MultiPhraseQuery.java =================================================================== --- src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/MultiPhraseQuery.java (working copy) @@ -21,10 +21,13 @@ import java.util.*; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultipleTermPositions; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.PositionsEnum; +import org.apache.lucene.index.TermRef; import org.apache.lucene.util.ToStringUtils; +import org.apache.lucene.util.PriorityQueue; +import org.apache.lucene.util.Bits; /** * MultiPhraseQuery is a generalized version of PhraseQuery, with an added @@ -162,27 +165,31 @@ if (termArrays.size() == 0) // optimize zero-term case return null; - TermPositions[] tps = new TermPositions[termArrays.size()]; - for (int i=0; i 1) - p = new MultipleTermPositions(reader, terms); - else - p = reader.termPositions(terms[0]); + final DocsEnum docsEnum; + if (terms.length > 1) { + docsEnum = new UnionDocsEnum(reader, terms); + } else { + docsEnum = reader.termDocsEnum(reader.getDeletedDocs(), + terms[0].field(), + new TermRef(terms[0].text())); + } - if (p == null) + if (docsEnum == null) { return null; + } - tps[i] = p; + docs[i] = docsEnum; } if (slop == 0) - return new ExactPhraseScorer(this, tps, getPositions(), similarity, + return new ExactPhraseScorer(this, docs, getPositions(), similarity, reader.norms(field)); else - return new SloppyPhraseScorer(this, tps, getPositions(), similarity, + return new SloppyPhraseScorer(this, docs, getPositions(), similarity, slop, reader.norms(field)); } @@ -371,3 +378,187 @@ return true; } } + +/** + * Takes the logical union of multiple DocsEnum iterators. + */ + +class UnionDocsEnum extends DocsEnum { + + private final static class DocsEnumWrapper { + int doc; + final DocsEnum docsEnum; + public DocsEnumWrapper(DocsEnum docsEnum) { + this.docsEnum = docsEnum; + } + } + + private static final class DocsQueue extends PriorityQueue { + DocsQueue(List docsEnums) throws IOException { + initialize(docsEnums.size()); + + Iterator i = docsEnums.iterator(); + while (i.hasNext()) { + DocsEnumWrapper docs = (DocsEnumWrapper) i.next(); + docs.doc = docs.docsEnum.next(); + if (docs.doc != DocsEnum.NO_MORE_DOCS) { + put(docs); + } + } + } + + final public DocsEnumWrapper peek() { + return (DocsEnumWrapper) top(); + } + + public final boolean lessThan(Object a, Object b) { + return ((DocsEnumWrapper) a).doc < ((DocsEnumWrapper) b).doc; + } + } + + private static final class IntQueue { + private int _arraySize = 16; + private int _index = 0; + private int _lastIndex = 0; + private int[] _array = new int[_arraySize]; + + final void add(int i) { + if (_lastIndex == _arraySize) + growArray(); + + _array[_lastIndex++] = i; + } + + final int next() { + return _array[_index++]; + } + + final void sort() { + Arrays.sort(_array, _index, _lastIndex); + } + + final void clear() { + _index = 0; + _lastIndex = 0; + } + + final int size() { + return (_lastIndex - _index); + } + + private void growArray() { + int[] newArray = new int[_arraySize * 2]; + System.arraycopy(_array, 0, newArray, 0, _arraySize); + _array = newArray; + _arraySize *= 2; + } + } + + private int _doc; + private int _freq; + private DocsQueue _queue; + private IntQueue _posList; + + private final UnionPositionsEnum unionPositionsEnum; + + public UnionDocsEnum(IndexReader indexReader, Term[] terms) throws IOException { + List docsEnums = new LinkedList(); + final Bits delDocs = indexReader.getDeletedDocs(); + + for (int i = 0; i < terms.length; i++) { + DocsEnum docs = indexReader.termDocsEnum(delDocs, + terms[i].field(), + new TermRef(terms[i].text())); + if (docs != null) { + docsEnums.add(new DocsEnumWrapper(docs)); + } + } + + _queue = new DocsQueue(docsEnums); + _posList = new IntQueue(); + unionPositionsEnum = new UnionPositionsEnum(); + } + + public PositionsEnum positions() { + return unionPositionsEnum; + } + + public final int next() throws IOException { + if (_queue.size() == 0) { + return NO_MORE_DOCS; + } + + // TODO: move this init into positions(): if the search + // doesn't need the positions for this doc then don't + // waste CPU merging them: + _posList.clear(); + _doc = _queue.peek().doc; + + // merge sort all positions together + DocsEnumWrapper docs; + do { + docs = _queue.peek(); + final PositionsEnum positions = docs.docsEnum.positions(); + + final int freq = docs.docsEnum.freq(); + for (int i = 0; i < freq; i++) { + _posList.add(positions.next()); + } + + docs.doc = docs.docsEnum.next(); + + if (docs.doc != NO_MORE_DOCS) { + _queue.adjustTop(); + } else { + _queue.pop(); + } + } while (_queue.size() > 0 && _queue.peek().doc == _doc); + + _posList.sort(); + _freq = _posList.size(); + + return _doc; + } + + private class UnionPositionsEnum extends PositionsEnum { + + public int next() { + return _posList.next(); + } + + public int getPayloadLength() { + throw new UnsupportedOperationException(); + } + + public byte[] getPayload(byte[] data, int offset) { + throw new UnsupportedOperationException(); + } + + public boolean hasPayload() { + throw new UnsupportedOperationException(); + } + } + + public final int advance(int target) throws IOException { + while (_queue.peek() != null && target > _queue.peek().doc) { + DocsEnumWrapper docs = (DocsEnumWrapper) _queue.pop(); + docs.doc = docs.docsEnum.advance(target); + if (docs.doc != NO_MORE_DOCS) { + _queue.put(docs); + } + } + return next(); + } + + public final int freq() { + return _freq; + } + + /** + * Not implemented. + * @throws UnsupportedOperationException + */ + public int read(int[] arg0, int[] arg1) throws IOException { + throw new UnsupportedOperationException(); + } +} Index: src/java/org/apache/lucene/search/MultiTermQuery.java =================================================================== --- src/java/org/apache/lucene/search/MultiTermQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/MultiTermQuery.java (working copy) @@ -25,6 +25,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermRef; import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.queryParser.QueryParser; // for javadoc @@ -98,24 +99,47 @@ private static class ScoringBooleanQueryRewrite extends RewriteMethod implements Serializable { public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException { - FilteredTermEnum enumerator = query.getEnum(reader); - BooleanQuery result = new BooleanQuery(true); - int count = 0; - try { - do { - Term t = enumerator.term(); + // nocommit + FilteredTermsEnum termsEnum = query.getTermsEnum(reader); + if (termsEnum != null) { + final String field = termsEnum.field(); + BooleanQuery result = new BooleanQuery(true); + int count = 0; + while(true) { + TermRef t = termsEnum.term(); if (t != null) { - TermQuery tq = new TermQuery(t); // found a match - tq.setBoost(query.getBoost() * enumerator.difference()); // set the boost + TermQuery tq = new TermQuery(new Term(field, t.toString())); // found a match + tq.setBoost(query.getBoost() * termsEnum.difference()); // set the boost result.add(tq, BooleanClause.Occur.SHOULD); // add to query count++; + termsEnum.next(); + } else { + break; } - } while (enumerator.next()); - } finally { - enumerator.close(); + } + query.incTotalNumberOfTerms(count); + return result; + } else { + // deprecated case + FilteredTermEnum enumerator = query.getEnum(reader); + BooleanQuery result = new BooleanQuery(true); + int count = 0; + try { + do { + Term t = enumerator.term(); + if (t != null) { + TermQuery tq = new TermQuery(t); // found a match + tq.setBoost(query.getBoost() * enumerator.difference()); // set the boost + result.add(tq, BooleanClause.Occur.SHOULD); // add to query + count++; + } + } while (enumerator.next()); + } finally { + enumerator.close(); + } + query.incTotalNumberOfTerms(count); + return result; } - query.incTotalNumberOfTerms(count); - return result; } // Make sure we are still a singleton even after deserializing @@ -224,32 +248,26 @@ final int termCountLimit = Math.min(BooleanQuery.getMaxClauseCount(), termCountCutoff); int docVisitCount = 0; - FilteredTermEnum enumerator = query.getEnum(reader); - try { + FilteredTermsEnum termsEnum = query.getTermsEnum(reader); + if (termsEnum != null) { + final String field = termsEnum.field(); while(true) { - Term t = enumerator.term(); + TermRef t = termsEnum.term(); if (t != null) { - pendingTerms.add(t); + pendingTerms.add(t.clone()); // Loading the TermInfo from the terms dict here // should not be costly, because 1) the // query/filter will load the TermInfo when it // runs, and 2) the terms dict has a cache: - docVisitCount += reader.docFreq(t); - } - - if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) { - // Too many terms -- make a filter. - Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query)); - result.setBoost(query.getBoost()); - return result; - } else if (!enumerator.next()) { + docVisitCount += reader.docFreq(field, t); + } else { // Enumeration is done, and we hit a small // enough number of terms & docs -- just make a // BooleanQuery, now Iterator it = pendingTerms.iterator(); BooleanQuery bq = new BooleanQuery(true); while(it.hasNext()) { - TermQuery tq = new TermQuery((Term) it.next()); + TermQuery tq = new TermQuery(new Term(field, ((TermRef) it.next()).toString())); bq.add(tq, BooleanClause.Occur.SHOULD); } // Strip scores @@ -258,19 +276,64 @@ query.incTotalNumberOfTerms(pendingTerms.size()); return result; } + + if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) { + // Too many terms -- make a filter. + Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query)); + result.setBoost(query.getBoost()); + return result; + } + termsEnum.next(); } - } finally { - enumerator.close(); + } else { + + // deprecated case + FilteredTermEnum enumerator = query.getEnum(reader); + try { + while(true) { + Term t = enumerator.term(); + if (t != null) { + pendingTerms.add(t); + // Loading the TermInfo from the terms dict here + // should not be costly, because 1) the + // query/filter will load the TermInfo when it + // runs, and 2) the terms dict has a cache: + docVisitCount += reader.docFreq(t); + } + + if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) { + // Too many terms -- make a filter. + Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query)); + result.setBoost(query.getBoost()); + return result; + } else if (!enumerator.next()) { + // Enumeration is done, and we hit a small + // enough number of terms & docs -- just make a + // BooleanQuery, now + Iterator it = pendingTerms.iterator(); + BooleanQuery bq = new BooleanQuery(true); + while(it.hasNext()) { + TermQuery tq = new TermQuery((Term) it.next()); + bq.add(tq, BooleanClause.Occur.SHOULD); + } + // Strip scores + Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq)); + result.setBoost(query.getBoost()); + query.incTotalNumberOfTerms(pendingTerms.size()); + return result; + } + } + } finally { + enumerator.close(); + } } } - @Override public int hashCode() { final int prime = 1279; return (int) (prime * termCountCutoff + Double.doubleToLongBits(docCountPercent)); } - @Override public boolean equals(Object obj) { if (this == obj) return true; @@ -346,9 +409,21 @@ return term; } - /** Construct the enumeration to be used, expanding the pattern term. */ - protected abstract FilteredTermEnum getEnum(IndexReader reader) - throws IOException; + /** Construct the enumeration to be used, expanding the + * pattern term. + * @deprecated Please override {@link #getTermsEnum} instead */ + protected FilteredTermEnum getEnum(IndexReader reader) + throws IOException { + return null; + } + + /** Construct the enumeration to be used, expanding the + * pattern term. + * nocommit in 3.x this will become abstract */ + protected FilteredTermsEnum getTermsEnum(IndexReader reader) + throws IOException { + return null; + } /** * Expert: Return the number of unique terms visited during execution of the query. Index: src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java =================================================================== --- src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (revision 822088) +++ src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (working copy) @@ -21,7 +21,11 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.Bits; import java.io.IOException; import java.util.BitSet; @@ -96,6 +100,7 @@ } abstract class TermGenerator { + // @deprecated public void generate(IndexReader reader, TermEnum enumerator) throws IOException { final int[] docs = new int[32]; final int[] freqs = new int[32]; @@ -126,6 +131,33 @@ termDocs.close(); } } + + public void generate(IndexReader reader, TermsEnum enumerator) throws IOException { + final int[] docs = new int[32]; + final int[] freqs = new int[32]; + int termCount = 0; + final Bits delDocs = reader.getDeletedDocs(); + while(true) { + termCount++; + DocsEnum docsEnum = enumerator.docs(delDocs); + while (true) { + final int count = docsEnum.read(docs, freqs); + if (count != 0) { + for(int i=0;i shortcut - if (enumerator.term() == null) + final FilteredTermsEnum termsEnum = query.getTermsEnum(reader); + if (termsEnum != null) { + // if enum is already ended, the enum is empty -> shortcut + if (termsEnum.term() == null) { return DocIdSet.EMPTY_DOCIDSET; + } // else fill into a OpenBitSet final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc()); new TermGenerator() { public void handleDoc(int doc) { bitSet.set(doc); } - }.generate(reader, enumerator); + }.generate(reader, termsEnum); return bitSet; - } finally { - enumerator.close(); + } else { + final TermEnum enumerator = query.getEnum(reader); + try { + // if current term in enum is null, the enum is empty -> shortcut + if (enumerator.term() == null) + return DocIdSet.EMPTY_DOCIDSET; + // else fill into a OpenBitSet + final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc()); + new TermGenerator() { + public void handleDoc(int doc) { + bitSet.set(doc); + } + }.generate(reader, enumerator); + return bitSet; + } finally { + enumerator.close(); + } } } Index: src/java/org/apache/lucene/search/NumericRangeQuery.java =================================================================== --- src/java/org/apache/lucene/search/NumericRangeQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/NumericRangeQuery.java (working copy) @@ -27,6 +27,8 @@ import org.apache.lucene.util.StringHelper; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.Terms; /** *

A {@link Query} that matches numeric values within a @@ -300,6 +302,10 @@ return new NumericRangeTermEnum(reader); } + protected FilteredTermsEnum getTermsEnum(final IndexReader reader) throws IOException { + return new NumericRangeTermsEnum(reader); + } + /** Returns the field name for this query */ public String getField() { return field; } @@ -373,7 +379,11 @@ * The ordering depends on how {@link NumericUtils#splitLongRange} and * {@link NumericUtils#splitIntRange} generates the sub-ranges. For * {@link MultiTermQuery} ordering is not relevant. + * + * @deprecated use NumericRangeTermsEnum instead */ + // nocommit -- can we remove this? only back compat + // concern would be subclasses of NRQ that invoke getEnum private final class NumericRangeTermEnum extends FilteredTermEnum { private final IndexReader reader; @@ -527,5 +537,169 @@ } } + + + /** + * Subclass of FilteredTermsEnum for enumerating all terms that match the + * sub-ranges for trie range queries, using flex API. + *

+ * WARNING: This term enumeration is not guaranteed to be always ordered by + * {@link Term#compareTo}. + * The ordering depends on how {@link NumericUtils#splitLongRange} and + * {@link NumericUtils#splitIntRange} generates the sub-ranges. For + * {@link MultiTermQuery} ordering is not relevant. + */ + private final class NumericRangeTermsEnum extends FilteredTermsEnum { + + private final IndexReader reader; + private final LinkedList rangeBounds = new LinkedList(); + private TermRef currentUpperBound = null; + + NumericRangeTermsEnum(final IndexReader reader) throws IOException { + this.reader = reader; + + switch (valSize) { + case 64: { + // lower + long minBound = Long.MIN_VALUE; + if (min instanceof Long) { + minBound = min.longValue(); + } else if (min instanceof Double) { + minBound = NumericUtils.doubleToSortableLong(min.doubleValue()); + } + if (!minInclusive && min != null) { + if (minBound == Long.MAX_VALUE) break; + minBound++; + } + + // upper + long maxBound = Long.MAX_VALUE; + if (max instanceof Long) { + maxBound = max.longValue(); + } else if (max instanceof Double) { + maxBound = NumericUtils.doubleToSortableLong(max.doubleValue()); + } + if (!maxInclusive && max != null) { + if (maxBound == Long.MIN_VALUE) break; + maxBound--; + } + + NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() { + //@Override + public final void addRange(String minPrefixCoded, String maxPrefixCoded) { + rangeBounds.add(minPrefixCoded); + rangeBounds.add(maxPrefixCoded); + } + }, precisionStep, minBound, maxBound); + break; + } + + case 32: { + // lower + int minBound = Integer.MIN_VALUE; + if (min instanceof Integer) { + minBound = min.intValue(); + } else if (min instanceof Float) { + minBound = NumericUtils.floatToSortableInt(min.floatValue()); + } + if (!minInclusive && min != null) { + if (minBound == Integer.MAX_VALUE) break; + minBound++; + } + + // upper + int maxBound = Integer.MAX_VALUE; + if (max instanceof Integer) { + maxBound = max.intValue(); + } else if (max instanceof Float) { + maxBound = NumericUtils.floatToSortableInt(max.floatValue()); + } + if (!maxInclusive && max != null) { + if (maxBound == Integer.MIN_VALUE) break; + maxBound--; + } + + NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() { + //@Override + public final void addRange(String minPrefixCoded, String maxPrefixCoded) { + rangeBounds.add(minPrefixCoded); + rangeBounds.add(maxPrefixCoded); + } + }, precisionStep, minBound, maxBound); + break; + } + + default: + // should never happen + throw new IllegalArgumentException("valSize must be 32 or 64"); + } + + // seek to first term + next(); + } + + //@Override + public float difference() { + return 1.0f; + } + + /** this is a dummy, it is not used by this class. */ + //@Override + protected boolean end() { + assert false; // should never be called + return current == null; + } + + public String field() { + return field; + } + + /** + * Compares if current upper bound is reached, + * this also updates the term count for statistics. + * In contrast to {@link FilteredTermEnum}, a return value + * of false ends iterating the current enum + * and forwards to the next sub-range. + */ + //@Override + protected boolean accept(TermRef term) { + return (term.compareTerm(currentUpperBound) <= 0); + } + + /** Increments the enumeration to the next element. True if one exists. */ + //@Override + public TermRef next() throws IOException { + // if a current term exists, the actual enum is initialized: + // try change to next term, if no such term exists, fall-through + if (current != null) { + assert actualEnum != null; + current = actualEnum.next(); + if (current != null && accept(current)) { + return current; + } + } + + // if all above fails, we go forward to the next enum, + // if one is available + current = null; + if (rangeBounds.size() < 2) { + assert rangeBounds.size() == 0; + return null; + } + + final TermRef lowerBound = new TermRef(rangeBounds.removeFirst()); + this.currentUpperBound = new TermRef(rangeBounds.removeFirst()); + + // this call recursively uses next(), if no valid term in + // next enum found. + // if this behavior is changed/modified in the superclass, + // this enum will not work anymore! + Terms terms = reader.fields().terms(field); + if (terms != null) { + init(terms.iterator(), lowerBound); + } + return current; + } + } } Index: src/java/org/apache/lucene/search/PhrasePositions.java =================================================================== --- src/java/org/apache/lucene/search/PhrasePositions.java (revision 822088) +++ src/java/org/apache/lucene/search/PhrasePositions.java (working copy) @@ -28,40 +28,43 @@ int position; // position in doc int count; // remaining pos in this doc int offset; // position in phrase - TermPositions tp; // stream of positions - PhrasePositions next; // used to make lists + final DocsEnum docs; // stream of docs + PositionsEnum positions; // positions in current doc + PhrasePositions next; // used to make lists boolean repeats; // there's other pp for same term (e.g. query="1st word 2nd word"~1) - PhrasePositions(TermPositions t, int o) { - tp = t; + PhrasePositions(DocsEnum docs, int o) { + this.docs = docs; offset = o; } final boolean next() throws IOException { // increments to next doc - if (!tp.next()) { - tp.close(); // close stream - doc = Integer.MAX_VALUE; // sentinel value + doc = docs.next(); + if (doc == docs.NO_MORE_DOCS) { return false; } - doc = tp.doc(); - position = 0; + positions = docs.positions(); + + // nocommit -- really needed? + //position = 0; + return true; } final boolean skipTo(int target) throws IOException { - if (!tp.skipTo(target)) { - tp.close(); // close stream - doc = Integer.MAX_VALUE; // sentinel value + doc = docs.advance(target); + if (doc == docs.NO_MORE_DOCS) { return false; } - doc = tp.doc(); - position = 0; + // nocommit -- really needed? + // position = 0; return true; } final void firstPosition() throws IOException { - count = tp.freq(); // read first pos + count = docs.freq(); // read first pos + positions = docs.positions(); nextPosition(); } @@ -73,7 +76,7 @@ */ final boolean nextPosition() throws IOException { if (count-- > 0) { // read subsequent pos's - position = tp.nextPosition() - offset; + position = positions.next() - offset; return true; } else return false; Index: src/java/org/apache/lucene/search/PhraseQuery.java =================================================================== --- src/java/org/apache/lucene/search/PhraseQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/PhraseQuery.java (working copy) @@ -22,10 +22,12 @@ import java.util.ArrayList; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Explanation.IDFExplanation; import org.apache.lucene.util.ToStringUtils; +import org.apache.lucene.util.Bits; /** A Query that matches documents containing a particular sequence of terms. * A PhraseQuery is built by QueryParser for input like "new york". @@ -143,20 +145,25 @@ if (terms.size() == 0) // optimize zero-term case return null; - TermPositions[] tps = new TermPositions[terms.size()]; + DocsEnum[] docs = new DocsEnum[terms.size()]; + final Bits delDocs = reader.getDeletedDocs(); for (int i = 0; i < terms.size(); i++) { - TermPositions p = reader.termPositions((Term)terms.get(i)); - if (p == null) + final Term t = (Term) terms.get(i); + DocsEnum docsEnum = reader.termDocsEnum(delDocs, + t.field(), + new TermRef(t.text())); + if (docsEnum == null) { return null; - tps[i] = p; + } + docs[i] = docsEnum; } if (slop == 0) // optimize exact case - return new ExactPhraseScorer(this, tps, getPositions(), similarity, + return new ExactPhraseScorer(this, docs, getPositions(), similarity, reader.norms(field)); else return - new SloppyPhraseScorer(this, tps, getPositions(), similarity, slop, + new SloppyPhraseScorer(this, docs, getPositions(), similarity, slop, reader.norms(field)); } Index: src/java/org/apache/lucene/search/PhraseScorer.java =================================================================== --- src/java/org/apache/lucene/search/PhraseScorer.java (revision 822088) +++ src/java/org/apache/lucene/search/PhraseScorer.java (working copy) @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.DocsEnum; /** Expert: Scoring functionality for phrase queries. *
A document is considered matching if it contains the phrase-query terms @@ -43,7 +43,7 @@ private float freq; //phrase frequency in current doc as computed by phraseFreq(). - PhraseScorer(Weight weight, TermPositions[] tps, int[] offsets, + PhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets, Similarity similarity, byte[] norms) { super(similarity); this.norms = norms; @@ -55,8 +55,8 @@ // reflects the phrase offset: pp.pos = tp.pos - offset. // this allows to easily identify a matching (exact) phrase // when all PhrasePositions have exactly the same position. - for (int i = 0; i < tps.length; i++) { - PhrasePositions pp = new PhrasePositions(tps[i], offsets[i]); + for (int i = 0; i < docs.length; i++) { + PhrasePositions pp = new PhrasePositions(docs[i], offsets[i]); if (last != null) { // add next to end of list last.next = pp; } else { @@ -65,7 +65,7 @@ last = pp; } - pq = new PhraseQueue(tps.length); // construct empty pq + pq = new PhraseQueue(docs.length); // construct empty pq first.doc = -1; } Index: src/java/org/apache/lucene/search/PrefixQuery.java =================================================================== --- src/java/org/apache/lucene/search/PrefixQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/PrefixQuery.java (working copy) @@ -41,8 +41,8 @@ /** Returns the prefix of this query. */ public Term getPrefix() { return prefix; } - protected FilteredTermEnum getEnum(IndexReader reader) throws IOException { - return new PrefixTermEnum(reader, prefix); + protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException { + return new PrefixTermsEnum(reader, prefix); } /** Prints a user-readable version of this query. */ Index: src/java/org/apache/lucene/search/PrefixTermEnum.java =================================================================== --- src/java/org/apache/lucene/search/PrefixTermEnum.java (revision 822088) +++ src/java/org/apache/lucene/search/PrefixTermEnum.java (working copy) @@ -29,6 +29,7 @@ * Term enumerations are always ordered by Term.compareTo(). Each term in * the enumeration is greater than all that precede it. * + * @deprecated Use {@link PrefixTermsEnum} instead. */ public class PrefixTermEnum extends FilteredTermEnum { Index: src/java/org/apache/lucene/search/PrefixTermsEnum.java =================================================================== --- src/java/org/apache/lucene/search/PrefixTermsEnum.java (revision 0) +++ src/java/org/apache/lucene/search/PrefixTermsEnum.java (revision 0) @@ -0,0 +1,77 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermRef; + +/** + * Subclass of FilteredTermEnum for enumerating all terms that match the + * specified prefix filter term. + *

+ * Term enumerations are always ordered by Term.compareTo(). Each term in + * the enumeration is greater than all that precede it. + * + * @deprecated Use {@link PrefixTermsEnum} instead. + */ +public class PrefixTermsEnum extends FilteredTermsEnum { + + private final Term prefix; + private final TermRef prefixRef; + private boolean end = false; + + public PrefixTermsEnum(IndexReader reader, Term prefix) throws IOException { + this.prefix = prefix; + Terms terms = reader.fields().terms(prefix.field()); + prefixRef = new TermRef(prefix.text()); + if (terms != null) { + init(terms.iterator(), prefixRef); + } else { + end = true; + } + } + + public String field() { + return prefix.field(); + } + + public float difference() { + return 1.0f; + } + + protected boolean end() { + return end; + } + + protected Term getPrefixTerm() { + return prefix; + } + + protected boolean accept(TermRef term) { + if (term.startsWith(prefixRef)) { + return true; + } else { + end = true; + return false; + } + } +} Index: src/java/org/apache/lucene/search/SloppyPhraseScorer.java =================================================================== --- src/java/org/apache/lucene/search/SloppyPhraseScorer.java (revision 822088) +++ src/java/org/apache/lucene/search/SloppyPhraseScorer.java (working copy) @@ -17,7 +17,7 @@ * limitations under the License. */ -import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.DocsEnum; import java.io.IOException; import java.util.HashMap; @@ -28,9 +28,9 @@ private PhrasePositions tmpPos[]; // for flipping repeating pps. private boolean checkedRepeats; - SloppyPhraseScorer(Weight weight, TermPositions[] tps, int[] offsets, Similarity similarity, + SloppyPhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets, Similarity similarity, int slop, byte[] norms) { - super(weight, tps, offsets, similarity, norms); + super(weight, docs, offsets, similarity, norms); this.slop = slop; } Index: src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- src/java/org/apache/lucene/search/TermQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -20,8 +20,9 @@ import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermRef; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Explanation.IDFExplanation; import org.apache.lucene.util.ToStringUtils; @@ -64,12 +65,12 @@ } public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - TermDocs termDocs = reader.termDocs(term); - - if (termDocs == null) + DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field(), new TermRef(term.text())); + if (docs == null) { return null; + } - return new TermScorer(this, termDocs, similarity, reader.norms(term.field())); + return new TermScorer(this, docs, similarity, reader.norms(term.field())); } public Explanation explain(IndexReader reader, int doc) Index: src/java/org/apache/lucene/search/TermRangeTermEnum.java =================================================================== --- src/java/org/apache/lucene/search/TermRangeTermEnum.java (revision 822088) +++ src/java/org/apache/lucene/search/TermRangeTermEnum.java (working copy) @@ -31,6 +31,7 @@ * Term enumerations are always ordered by Term.compareTo(). Each term in * the enumeration is greater than all that precede it. * @since 2.9 + * @deprecated Please switch to {@link TermRangeTermsEnum} */ public class TermRangeTermEnum extends FilteredTermEnum { Index: src/java/org/apache/lucene/search/TermRangeTermsEnum.java =================================================================== --- src/java/org/apache/lucene/search/TermRangeTermsEnum.java (revision 0) +++ src/java/org/apache/lucene/search/TermRangeTermsEnum.java (revision 0) @@ -0,0 +1,157 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.text.Collator; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.TermRef; +import org.apache.lucene.index.Terms; +//import org.apache.lucene.index.Term; +import org.apache.lucene.util.StringHelper; + +/** + * Subclass of FilteredTermEnum for enumerating all terms that match the + * specified range parameters. + *

+ * Term enumerations are always ordered by Term.compareTo(). Each term in + * the enumeration is greater than all that precede it. + */ +public class TermRangeTermsEnum extends FilteredTermsEnum { + + private Collator collator; + private boolean end; + private String field; + private String upperTermText; + private String lowerTermText; + private boolean includeLower; + private boolean includeUpper; + final private TermRef lowerTermRef; + final private TermRef upperTermRef; + private boolean testLower; + + /** + * Enumerates all terms greater/equal than lowerTerm + * but less/equal than upperTerm. + * + * If an endpoint is null, it is said to be "open". Either or both + * endpoints may be open. Open endpoints may not be exclusive + * (you can't select all but the first or last term without + * explicitly specifying the term to exclude.) + * + * @param reader + * @param field + * An interned field that holds both lower and upper terms. + * @param lowerTermText + * The term text at the lower end of the range + * @param upperTermText + * The term text at the upper end of the range + * @param includeLower + * If true, the lowerTerm is included in the range. + * @param includeUpper + * If true, the upperTerm is included in the range. + * @param collator + * The collator to use to collate index Terms, to determine their + * membership in the range bounded by lowerTerm and + * upperTerm. + * + * @throws IOException + */ + public TermRangeTermsEnum(IndexReader reader, String field, String lowerTermText, String upperTermText, + boolean includeLower, boolean includeUpper, Collator collator) throws IOException { + this.collator = collator; + this.upperTermText = upperTermText; + this.lowerTermText = lowerTermText; + this.includeLower = includeLower; + this.includeUpper = includeUpper; + this.field = StringHelper.intern(field); + // do a little bit of normalization... + // open ended range queries should always be inclusive. + if (this.lowerTermText == null) { + this.lowerTermText = ""; + this.includeLower = true; + } + lowerTermRef = new TermRef(this.lowerTermText); + + if (this.upperTermText == null) { + this.includeUpper = true; + upperTermRef = null; + } else { + upperTermRef = new TermRef(upperTermText); + } + + + String startTermText = collator == null ? this.lowerTermText : ""; + Terms terms = reader.fields().terms(field); + if (terms != null) { + testLower = !includeLower; + // init will invoke accept; we only need to test lower + // in this case + init(terms.iterator(), lowerTermRef); + testLower = false; + } else { + end = true; + } + } + + public float difference() { + return 1.0f; + } + + protected boolean end() { + return end; + } + + public String field() { + return field; + } + + protected boolean accept(TermRef term) { + if (collator == null) { + // Use Unicode code point ordering + if (!testLower || !term.termEquals(lowerTermRef)) { + if (upperTermRef != null) { + final int cmp = upperTermRef.compareTerm(term); + /* + * if beyond the upper term, or is exclusive and this is equal to + * the upper term, break out + */ + if ((cmp < 0) || + (!includeUpper && cmp==0)) { + end = true; + return false; + } + } + return true; + } + } else { + if ((includeLower + ? collator.compare(term.toString(), lowerTermText) >= 0 + : collator.compare(term.toString(), lowerTermText) > 0) + && (upperTermText == null + || (includeUpper + ? collator.compare(term.toString(), upperTermText) <= 0 + : collator.compare(term.toString(), upperTermText) < 0))) { + return true; + } + end = true; + } + return false; + } +} Index: src/java/org/apache/lucene/search/TermScorer.java =================================================================== --- src/java/org/apache/lucene/search/TermScorer.java (revision 822088) +++ src/java/org/apache/lucene/search/TermScorer.java (working copy) @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.DocsEnum; /** Expert: A Scorer for documents matching a Term. */ @@ -28,7 +28,7 @@ private static final float[] SIM_NORM_DECODER = Similarity.getNormDecoder(); private Weight weight; - private TermDocs termDocs; + private DocsEnum docsEnum; private byte[] norms; private float weightValue; private int doc = -1; @@ -54,10 +54,10 @@ * @param norms * The field norms of the document fields for the Term. */ - TermScorer(Weight weight, TermDocs td, Similarity similarity, byte[] norms) { + TermScorer(Weight weight, DocsEnum td, Similarity similarity, byte[] norms) { super(similarity); this.weight = weight; - this.termDocs = td; + this.docsEnum = td; this.norms = norms; this.weightValue = weight.getValue(); @@ -81,17 +81,17 @@ // firstDocID is ignored since nextDoc() sets 'doc' protected boolean score(Collector c, int end, int firstDocID) throws IOException { + //System.out.println("top score " + firstDocID + " max=" + pointerMax); c.setScorer(this); while (doc < end) { // for docs in window c.collect(doc); // collect score - + //System.out.println("done collect"); if (++pointer >= pointerMax) { - pointerMax = termDocs.read(docs, freqs); // refill buffers + pointerMax = docsEnum.read(docs, freqs); // refill buffers if (pointerMax != 0) { pointer = 0; } else { - termDocs.close(); // close stream - doc = Integer.MAX_VALUE; // set to sentinel value + doc = NO_MORE_DOCS; // set to sentinel value return false; } } @@ -122,25 +122,28 @@ * The iterator over the matching documents is buffered using * {@link TermDocs#read(int[],int[])}. * - * @return the document matching the query or -1 if there are no more documents. + * @return the document matching the query or NO_MORE_DOCS if there are no more documents. */ public int nextDoc() throws IOException { + //System.out.println("ts.nextDoc pointer=" + pointer + " max=" + pointerMax + " this=" + this + " docsEnum=" + docsEnum); pointer++; if (pointer >= pointerMax) { - pointerMax = termDocs.read(docs, freqs); // refill buffer + pointerMax = docsEnum.read(docs, freqs); // refill buffer + //System.out.println("ts set max=" + pointerMax); if (pointerMax != 0) { pointer = 0; } else { - termDocs.close(); // close stream + //System.out.println("ts no more docs"); return doc = NO_MORE_DOCS; } } doc = docs[pointer]; + assert doc != NO_MORE_DOCS; return doc; } public float score() { - assert doc != -1; + assert doc != NO_MORE_DOCS; int f = freqs[pointer]; float raw = // compute tf(f)*weight f < SCORE_CACHE_SIZE // check cache @@ -153,7 +156,7 @@ /** * Skips to the first match beyond the current whose document number is * greater than or equal to a given target.
- * The implementation uses {@link TermDocs#skipTo(int)}. + * The implementation uses {@link DocsEnum#advance(int)}. * * @param target * The target document number. @@ -167,11 +170,11 @@ /** * Advances to the first match beyond the current whose document number is * greater than or equal to a given target.
- * The implementation uses {@link TermDocs#skipTo(int)}. + * The implementation uses {@link DocsEnum#adnvace(int)}. * * @param target * The target document number. - * @return the matching document or -1 if none exist. + * @return the matching document or NO_MORE_DOCS if none exist. */ public int advance(int target) throws IOException { // first scan in cache @@ -181,13 +184,14 @@ } } - // not found in cache, seek underlying stream - boolean result = termDocs.skipTo(target); - if (result) { + // not found in readahead cache, seek underlying stream + int newDoc = docsEnum.advance(target); + //System.out.println("ts.advance docsEnum=" + docsEnum); + if (newDoc != DocsEnum.NO_MORE_DOCS) { pointerMax = 1; pointer = 0; - docs[pointer] = doc = termDocs.doc(); - freqs[pointer] = termDocs.freq(); + docs[pointer] = doc = newDoc; + freqs[pointer] = docsEnum.freq(); } else { doc = NO_MORE_DOCS; } @@ -209,15 +213,11 @@ pointer++; } if (tf == 0) { - if (termDocs.skipTo(doc)) - { - if (termDocs.doc() == doc) - { - tf = termDocs.freq(); - } - } + int newDoc = docsEnum.advance(doc); + if (newDoc == doc) { + tf = docsEnum.freq(); + } } - termDocs.close(); tfExplanation.setValue(getSimilarity().tf(tf)); tfExplanation.setDescription("tf(termFreq("+query.getTerm()+")="+tf+")"); @@ -225,5 +225,6 @@ } /** Returns a string representation of this TermScorer. */ - public String toString() { return "scorer(" + weight + ")"; } + // nocommit + //public String toString() { return "scorer(" + weight + ")"; } } Index: src/java/org/apache/lucene/search/WildcardQuery.java =================================================================== --- src/java/org/apache/lucene/search/WildcardQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/WildcardQuery.java (working copy) @@ -45,6 +45,10 @@ this.termContainsWildcard = (term.text().indexOf('*') != -1) || (term.text().indexOf('?') != -1); } + protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException { + return new WildcardTermsEnum(reader, getTerm()); + } + protected FilteredTermEnum getEnum(IndexReader reader) throws IOException { return new WildcardTermEnum(reader, getTerm()); } Index: src/java/org/apache/lucene/search/WildcardTermEnum.java =================================================================== --- src/java/org/apache/lucene/search/WildcardTermEnum.java (revision 822088) +++ src/java/org/apache/lucene/search/WildcardTermEnum.java (working copy) @@ -30,6 +30,7 @@ * the enumeration is greater than all that precede it. * * @version $Id$ + * @deprecated Please use {@link WildcardTermsEnum} instead. */ public class WildcardTermEnum extends FilteredTermEnum { final Term searchTerm; Index: src/java/org/apache/lucene/search/WildcardTermsEnum.java =================================================================== --- src/java/org/apache/lucene/search/WildcardTermsEnum.java (revision 0) +++ src/java/org/apache/lucene/search/WildcardTermsEnum.java (revision 0) @@ -0,0 +1,204 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermRef; + +/** + * Subclass of FilteredTermEnum for enumerating all terms that match the + * specified wildcard filter term. + *

+ * Term enumerations are always ordered by Term.compareTo(). Each term in + * the enumeration is greater than all that precede it. + * + * @version $Id: WildcardTermEnum.java 783371 2009-06-10 14:39:56Z mikemccand $ + */ +public class WildcardTermsEnum extends FilteredTermsEnum { + final Term searchTerm; + final String field; + final String text; + final String pre; + final int preLen; + private boolean end; + private final TermRef preTermRef; + + /** + * Creates a new WildcardTermEnum. + *

+ * After calling the constructor the enumeration is already pointing to the first + * valid term if such a term exists. + */ + public WildcardTermsEnum(IndexReader reader, Term term) throws IOException { + super(); + searchTerm = term; + field = searchTerm.field(); + final String searchTermText = searchTerm.text(); + + final int sidx = searchTermText.indexOf(WILDCARD_STRING); + final int cidx = searchTermText.indexOf(WILDCARD_CHAR); + int idx = sidx; + if (idx == -1) { + idx = cidx; + } + else if (cidx >= 0) { + idx = Math.min(idx, cidx); + } + pre = idx != -1?searchTerm.text().substring(0,idx): ""; + + preLen = pre.length(); + text = searchTermText.substring(preLen); + preTermRef = new TermRef(pre); + + Terms terms = reader.fields().terms(searchTerm.field()); + if (terms != null) { + init(terms.iterator(), preTermRef); + } else { + end = true; + } + } + + public String field() { + return searchTerm.field(); + } + + protected final boolean accept(TermRef term) { + if (term.startsWith(preTermRef)) { + // TODO: would be better, but trickier, to not have to + // build intermediate String (ie check wildcard matching + // directly on UTF8) + final String searchText = term.toString(); + return wildcardEquals(text, 0, searchText, preLen); + } + end = true; + return false; + } + + public float difference() { + return 1.0f; + } + + public final boolean end() { + return end; + } + + /******************************************** + * String equality with support for wildcards + ********************************************/ + + public static final char WILDCARD_STRING = '*'; + public static final char WILDCARD_CHAR = '?'; + + /** + * Determines if a word matches a wildcard pattern. + * Work released by Granta Design Ltd after originally being done on + * company time. + */ + public static final boolean wildcardEquals(String pattern, int patternIdx, + String string, int stringIdx) + { + int p = patternIdx; + + for (int s = stringIdx; ; ++p, ++s) + { + // End of string yet? + boolean sEnd = (s >= string.length()); + // End of pattern yet? + boolean pEnd = (p >= pattern.length()); + + // If we're looking at the end of the string... + if (sEnd) + { + // Assume the only thing left on the pattern is/are wildcards + boolean justWildcardsLeft = true; + + // Current wildcard position + int wildcardSearchPos = p; + // While we haven't found the end of the pattern, + // and haven't encountered any non-wildcard characters + while (wildcardSearchPos < pattern.length() && justWildcardsLeft) + { + // Check the character at the current position + char wildchar = pattern.charAt(wildcardSearchPos); + + // If it's not a wildcard character, then there is more + // pattern information after this/these wildcards. + if (wildchar != WILDCARD_CHAR && wildchar != WILDCARD_STRING) + { + justWildcardsLeft = false; + } + else + { + // to prevent "cat" matches "ca??" + if (wildchar == WILDCARD_CHAR) { + return false; + } + + // Look at the next character + wildcardSearchPos++; + } + } + + // This was a prefix wildcard search, and we've matched, so + // return true. + if (justWildcardsLeft) + { + return true; + } + } + + // If we've gone past the end of the string, or the pattern, + // return false. + if (sEnd || pEnd) + { + break; + } + + // Match a single character, so continue. + if (pattern.charAt(p) == WILDCARD_CHAR) + { + continue; + } + + // + if (pattern.charAt(p) == WILDCARD_STRING) + { + // Look at the character beyond the '*'. + ++p; + // Examine the string, starting at the last character. + for (int i = string.length(); i >= s; --i) + { + if (wildcardEquals(pattern, p, string, i)) + { + return true; + } + } + break; + } + if (pattern.charAt(p) != string.charAt(s)) + { + break; + } + } + return false; + } +} Index: src/java/org/apache/lucene/search/function/ValueSourceQuery.java =================================================================== --- src/java/org/apache/lucene/search/function/ValueSourceQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/function/ValueSourceQuery.java (working copy) @@ -18,9 +18,9 @@ */ import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.TermDocs; import org.apache.lucene.search.*; import org.apache.lucene.util.ToStringUtils; +import org.apache.lucene.util.Bits; import java.io.IOException; import java.util.Set; @@ -113,7 +113,8 @@ private final ValueSourceWeight weight; private final float qWeight; private final DocValues vals; - private final TermDocs termDocs; + private final Bits delDocs; + private final int maxDoc; private int doc = -1; // constructor @@ -123,21 +124,29 @@ this.qWeight = w.getValue(); // this is when/where the values are first created. vals = valSrc.getValues(reader); - termDocs = reader.termDocs(null); + delDocs = reader.getDeletedDocs(); + maxDoc = reader.maxDoc(); } /** @deprecated use {@link #nextDoc()} instead. */ public boolean next() throws IOException { - return termDocs.next(); + return nextDoc() != NO_MORE_DOCS; } public int nextDoc() throws IOException { - return doc = termDocs.next() ? termDocs.doc() : NO_MORE_DOCS; + doc++; + while(delDocs != null && doc < maxDoc && delDocs.get(doc)) { + doc++; + } + if (doc == maxDoc) { + doc = NO_MORE_DOCS; + } + return doc; } /** @deprecated use {@link #docID()} instead. */ public int doc() { - return termDocs.doc(); + return doc; } public int docID() { @@ -146,16 +155,17 @@ /*(non-Javadoc) @see org.apache.lucene.search.Scorer#score() */ public float score() throws IOException { - return qWeight * vals.floatVal(termDocs.doc()); + return qWeight * vals.floatVal(doc); } /** @deprecated use {@link #advance(int)} instead. */ public boolean skipTo(int target) throws IOException { - return termDocs.skipTo(target); + return advance(target) != NO_MORE_DOCS; } public int advance(int target) throws IOException { - return doc = termDocs.skipTo(target) ? termDocs.doc() : NO_MORE_DOCS; + doc = target-1; + return nextDoc(); } /*(non-Javadoc) @see org.apache.lucene.search.Scorer#explain(int) */ Index: src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java =================================================================== --- src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (working copy) @@ -19,7 +19,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.PositionsEnum; import org.apache.lucene.search.Searcher; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -80,14 +80,14 @@ protected class PayloadTermSpanScorer extends SpanScorer { // TODO: is this the best way to allocate this? protected byte[] payload = new byte[256]; - protected TermPositions positions; protected float payloadScore; protected int payloadsSeen; + private final TermSpans termSpans; public PayloadTermSpanScorer(TermSpans spans, Weight weight, Similarity similarity, byte[] norms) throws IOException { super(spans, weight, similarity, norms); - positions = spans.getPositions(); + termSpans = spans; } protected boolean setFreqCurrentDoc() throws IOException { @@ -112,7 +112,8 @@ } protected void processPayload(Similarity similarity) throws IOException { - if (positions.isPayloadAvailable()) { + final PositionsEnum positions = termSpans.getPositions(); + if (positions.hasPayload()) { payload = positions.getPayload(payload, 0); payloadScore = function.currentScore(doc, term.field(), spans.start(), spans.end(), payloadsSeen, payloadScore, Index: src/java/org/apache/lucene/search/spans/SpanTermQuery.java =================================================================== --- src/java/org/apache/lucene/search/spans/SpanTermQuery.java (revision 822088) +++ src/java/org/apache/lucene/search/spans/SpanTermQuery.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermRef; import org.apache.lucene.util.ToStringUtils; import java.io.IOException; @@ -85,7 +86,9 @@ } public Spans getSpans(final IndexReader reader) throws IOException { - return new TermSpans(reader.termPositions(term), term); + return new TermSpans(reader.termDocsEnum(reader.getDeletedDocs(), + term.field(), + new TermRef(term.text())), term); } } Index: src/java/org/apache/lucene/search/spans/TermSpans.java =================================================================== --- src/java/org/apache/lucene/search/spans/TermSpans.java (revision 822088) +++ src/java/org/apache/lucene/search/spans/TermSpans.java (working copy) @@ -17,7 +17,8 @@ import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.PositionsEnum; import java.io.IOException; import java.util.Collections; @@ -28,47 +29,46 @@ * Public for extension only */ public class TermSpans extends Spans { - protected TermPositions positions; - protected Term term; + protected final DocsEnum docs; + protected PositionsEnum positions; + protected final Term term; protected int doc; protected int freq; protected int count; protected int position; - - public TermSpans(TermPositions positions, Term term) throws IOException { - - this.positions = positions; + public TermSpans(DocsEnum docs, Term term) throws IOException { + this.docs = docs; this.term = term; doc = -1; } public boolean next() throws IOException { if (count == freq) { - if (!positions.next()) { - doc = Integer.MAX_VALUE; + doc = docs.next(); + if (doc == DocsEnum.NO_MORE_DOCS) { return false; } - doc = positions.doc(); - freq = positions.freq(); + freq = docs.freq(); + positions = docs.positions(); count = 0; } - position = positions.nextPosition(); + position = positions.next(); count++; return true; } public boolean skipTo(int target) throws IOException { - if (!positions.skipTo(target)) { - doc = Integer.MAX_VALUE; + doc = docs.advance(target); + if (doc == DocsEnum.NO_MORE_DOCS) { return false; } - doc = positions.doc(); - freq = positions.freq(); + freq = docs.freq(); count = 0; + positions = docs.positions(); - position = positions.nextPosition(); + position = positions.next(); count++; return true; @@ -95,7 +95,7 @@ // TODO: Remove warning after API has been finalized public boolean isPayloadAvailable() { - return positions.isPayloadAvailable(); + return positions.hasPayload(); } public String toString() { @@ -103,8 +103,7 @@ (doc == -1 ? "START" : (doc == Integer.MAX_VALUE) ? "END" : doc + "-" + position); } - - public TermPositions getPositions() { + public PositionsEnum getPositions() { return positions; } } Index: src/java/org/apache/lucene/store/Directory.java =================================================================== --- src/java/org/apache/lucene/store/Directory.java (revision 822088) +++ src/java/org/apache/lucene/store/Directory.java (working copy) @@ -19,8 +19,6 @@ import java.io.IOException; -import org.apache.lucene.index.IndexFileNameFilter; - /** A Directory is a flat list of files. Files may be written once, when they * are created. Once a file is created it may only be opened for read, or * deleted. Random access is permitted both when reading and writing. @@ -158,6 +156,9 @@ return this.toString(); } + // nocommit -- note runtime change that all files are + // copied + /** * Copy contents of a directory src to a directory dest. * If a file in src already exists in dest then the @@ -168,9 +169,8 @@ * are undefined and you could easily hit a * FileNotFoundException. * - *

NOTE: this method only copies files that look - * like index files (ie, have extensions matching the - * known extensions of index files). + *

NOTE: this method copies all files, not only + * files that look like index files * * @param src source directory * @param dest destination directory @@ -180,14 +180,9 @@ public static void copy(Directory src, Directory dest, boolean closeDirSrc) throws IOException { final String[] files = src.listAll(); - IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); - byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE]; for (int i = 0; i < files.length; i++) { - if (!filter.accept(null, files[i])) - continue; - IndexOutput os = null; IndexInput is = null; try { Index: src/java/org/apache/lucene/store/FileSwitchDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FileSwitchDirectory.java (revision 822088) +++ src/java/org/apache/lucene/store/FileSwitchDirectory.java (working copy) @@ -18,8 +18,6 @@ */ import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Set; /** Index: src/java/org/apache/lucene/store/RAMDirectory.java =================================================================== --- src/java/org/apache/lucene/store/RAMDirectory.java (revision 822088) +++ src/java/org/apache/lucene/store/RAMDirectory.java (working copy) @@ -19,7 +19,6 @@ import java.io.IOException; import java.io.FileNotFoundException; -import java.io.File; import java.io.Serializable; import java.util.HashMap; import java.util.Iterator; @@ -195,7 +194,8 @@ file = (RAMFile)fileMap.get(name); } if (file == null) - throw new FileNotFoundException(name); + // nocommit + throw new FileNotFoundException(name + " dir=" + this); return new RAMInputStream(file); } Index: src/java/org/apache/lucene/util/ArrayUtil.java =================================================================== --- src/java/org/apache/lucene/util/ArrayUtil.java (revision 822088) +++ src/java/org/apache/lucene/util/ArrayUtil.java (working copy) @@ -204,6 +204,29 @@ return grow(array, 1 + array.length); } + public static char[] shrink(char[] array, int targetSize) { + final int newSize = getShrinkSize(array.length, targetSize); + if (newSize != array.length) { + char[] newArray = new char[newSize]; + System.arraycopy(array, 0, newArray, 0, newSize); + return newArray; + } else + return array; + } + + public static char[] grow(char[] array, int minSize) { + if (array.length < minSize) { + char[] newArray = new char[getNextSize(minSize)]; + System.arraycopy(array, 0, newArray, 0, array.length); + return newArray; + } else + return array; + } + + public static char[] grow(char[] array) { + return grow(array, 1 + array.length); + } + public static byte[] shrink(byte[] array, int targetSize) { final int newSize = getShrinkSize(array.length, targetSize); if (newSize != array.length) { Index: src/java/org/apache/lucene/util/BitUtil.java =================================================================== --- src/java/org/apache/lucene/util/BitUtil.java (revision 822088) +++ src/java/org/apache/lucene/util/BitUtil.java (working copy) @@ -801,7 +801,7 @@ } /** returns the next highest power of two, or the current value if it's already a power of two or zero*/ - public static long nextHighestPowerOfTwo(long v) { + public static long nextHighestPowerOfTwo(long v) { v--; v |= v >> 1; v |= v >> 2; @@ -813,4 +813,26 @@ return v; } + /** Returns the smallest non negative p such that a given value < (2**(p+1)) + * This differs from (63 - java.lang.Long.numberOfLeadingZeros(v)) + * for non positive given values. + */ + public static int logNextHigherPowerOfTwo(long v) { + long vinput = v; // only for assertions below. + int p = 0; + while (v >= (1 << 8)) { + v >>= 8; + p += 8; + } + while (v >= (1 << 1)) { + v >>= 1; + p++; + } + assert (p <= 62) : p; + assert (p == 62) || (vinput < (1L << (p + 1))) : "p " + p + ", vinput " + vinput; + assert (p == 0) || (vinput >= (1L << p)) : "p " + p + ", vinput " + vinput; + assert (vinput <= 0) || (p == (63 - java.lang.Long.numberOfLeadingZeros(vinput))) : "p " + p + ", vinput " + vinput; + return p; + } + } Index: src/java/org/apache/lucene/util/BitVector.java =================================================================== --- src/java/org/apache/lucene/util/BitVector.java (revision 822088) +++ src/java/org/apache/lucene/util/BitVector.java (working copy) @@ -35,7 +35,7 @@ @version $Id$ */ -public final class BitVector implements Cloneable { +public final class BitVector implements Cloneable, Bits { private byte[] bits; private int size; Index: src/java/org/apache/lucene/util/Bits.java =================================================================== --- src/java/org/apache/lucene/util/Bits.java (revision 0) +++ src/java/org/apache/lucene/util/Bits.java (revision 0) @@ -0,0 +1,22 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public interface Bits { + public boolean get(int index); +} Index: src/java/org/apache/lucene/util/NumericUtils.java =================================================================== --- src/java/org/apache/lucene/util/NumericUtils.java (revision 822088) +++ src/java/org/apache/lucene/util/NumericUtils.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.document.NumericField; // for javadocs import org.apache.lucene.search.NumericRangeQuery; // for javadocs import org.apache.lucene.search.NumericRangeFilter; // for javadocs +import org.apache.lucene.index.TermRef; /** * This is a helper class to generate prefix-encoded representations for numerical values @@ -219,6 +220,26 @@ return (sortableBits << shift) ^ 0x8000000000000000L; } + public static long prefixCodedToLong(final TermRef term) { + final int shift = term.bytes[term.offset]-SHIFT_START_LONG; + if (shift>63 || shift<0) + throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really an INT?)"); + long sortableBits = 0L; + final int limit = term.offset + term.length; + for (int i=term.offset+1; i31 || shift<0) + throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really an INT?)"); + int sortableBits = 0; + final int limit = term.offset + term.length; + for (int i=term.offset+1; idouble value to a sortable signed long. * The value is converted by getting their IEEE 754 floating-point "double format" Index: src/java/org/apache/lucene/util/UnicodeUtil.java =================================================================== --- src/java/org/apache/lucene/util/UnicodeUtil.java (revision 822088) +++ src/java/org/apache/lucene/util/UnicodeUtil.java (working copy) @@ -73,14 +73,16 @@ private static final long HALF_MASK = 0x3FFL; public static final class UTF8Result { - public byte[] result = new byte[10]; + public byte[] result; public int length; + public UTF8Result() { + result = new byte[10]; + } + public void setLength(int newLength) { if (result.length < newLength) { - byte[] newArray = new byte[(int) (1.5*newLength)]; - System.arraycopy(result, 0, newArray, 0, length); - result = newArray; + result = ArrayUtil.grow(result, newLength); } length = newLength; } @@ -91,12 +93,15 @@ public int[] offsets = new int[10]; public int length; + /* + public String toString() { + return new String(result, 0, length); + } + */ + public void setLength(int newLength) { - if (result.length < newLength) { - char[] newArray = new char[(int) (1.5*newLength)]; - System.arraycopy(result, 0, newArray, 0, length); - result = newArray; - } + if (result.length < newLength) + result = ArrayUtil.grow(result, newLength); length = newLength; } @@ -104,6 +109,13 @@ setLength(other.length); System.arraycopy(other.result, 0, result, 0, length); } + + public void copyText(String other) { + final int otherLength = other.length(); + setLength(otherLength); + other.getChars(0, otherLength, result, 0); + length = otherLength; + } } /** Encode characters from a char[] source, starting at Index: src/java/org/apache/lucene/util/pfor/For10Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For10Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For10Decompress.java (revision 0) @@ -0,0 +1,82 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For10Decompress extends ForDecompress { + static final int numFrameBits = 10; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 10) & mask; + output[2 + outputOffset] = (intValue0 >>> 20) & mask; + output[3 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[4 + outputOffset] = (intValue1 >>> 8) & mask; + output[5 + outputOffset] = (intValue1 >>> 18) & mask; + output[6 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[7 + outputOffset] = (intValue2 >>> 6) & mask; + output[8 + outputOffset] = (intValue2 >>> 16) & mask; + output[9 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[10 + outputOffset] = (intValue3 >>> 4) & mask; + output[11 + outputOffset] = (intValue3 >>> 14) & mask; + output[12 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[13 + outputOffset] = (intValue4 >>> 2) & mask; + output[14 + outputOffset] = (intValue4 >>> 12) & mask; + output[15 + outputOffset] = intValue4 >>> 22; + output[16 + outputOffset] = intValue5 & mask; + output[17 + outputOffset] = (intValue5 >>> 10) & mask; + output[18 + outputOffset] = (intValue5 >>> 20) & mask; + output[19 + outputOffset] = ((intValue5 >>> 30) | (intValue6 << 2)) & mask; + output[20 + outputOffset] = (intValue6 >>> 8) & mask; + output[21 + outputOffset] = (intValue6 >>> 18) & mask; + output[22 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[23 + outputOffset] = (intValue7 >>> 6) & mask; + output[24 + outputOffset] = (intValue7 >>> 16) & mask; + output[25 + outputOffset] = ((intValue7 >>> 26) | (intValue8 << 6)) & mask; + output[26 + outputOffset] = (intValue8 >>> 4) & mask; + output[27 + outputOffset] = (intValue8 >>> 14) & mask; + output[28 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[29 + outputOffset] = (intValue9 >>> 2) & mask; + output[30 + outputOffset] = (intValue9 >>> 12) & mask; + output[31 + outputOffset] = intValue9 >>> 22; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For11Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For11Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For11Decompress.java (revision 0) @@ -0,0 +1,83 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For11Decompress extends ForDecompress { + static final int numFrameBits = 11; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 11) & mask; + output[2 + outputOffset] = ((intValue0 >>> 22) | (intValue1 << 10)) & mask; + output[3 + outputOffset] = (intValue1 >>> 1) & mask; + output[4 + outputOffset] = (intValue1 >>> 12) & mask; + output[5 + outputOffset] = ((intValue1 >>> 23) | (intValue2 << 9)) & mask; + output[6 + outputOffset] = (intValue2 >>> 2) & mask; + output[7 + outputOffset] = (intValue2 >>> 13) & mask; + output[8 + outputOffset] = ((intValue2 >>> 24) | (intValue3 << 8)) & mask; + output[9 + outputOffset] = (intValue3 >>> 3) & mask; + output[10 + outputOffset] = (intValue3 >>> 14) & mask; + output[11 + outputOffset] = ((intValue3 >>> 25) | (intValue4 << 7)) & mask; + output[12 + outputOffset] = (intValue4 >>> 4) & mask; + output[13 + outputOffset] = (intValue4 >>> 15) & mask; + output[14 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[15 + outputOffset] = (intValue5 >>> 5) & mask; + output[16 + outputOffset] = (intValue5 >>> 16) & mask; + output[17 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask; + output[18 + outputOffset] = (intValue6 >>> 6) & mask; + output[19 + outputOffset] = (intValue6 >>> 17) & mask; + output[20 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[21 + outputOffset] = (intValue7 >>> 7) & mask; + output[22 + outputOffset] = (intValue7 >>> 18) & mask; + output[23 + outputOffset] = ((intValue7 >>> 29) | (intValue8 << 3)) & mask; + output[24 + outputOffset] = (intValue8 >>> 8) & mask; + output[25 + outputOffset] = (intValue8 >>> 19) & mask; + output[26 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[27 + outputOffset] = (intValue9 >>> 9) & mask; + output[28 + outputOffset] = (intValue9 >>> 20) & mask; + output[29 + outputOffset] = ((intValue9 >>> 31) | (intValue10 << 1)) & mask; + output[30 + outputOffset] = (intValue10 >>> 10) & mask; + output[31 + outputOffset] = intValue10 >>> 21; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For12Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For12Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For12Decompress.java (revision 0) @@ -0,0 +1,84 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For12Decompress extends ForDecompress { + static final int numFrameBits = 12; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 12) & mask; + output[2 + outputOffset] = ((intValue0 >>> 24) | (intValue1 << 8)) & mask; + output[3 + outputOffset] = (intValue1 >>> 4) & mask; + output[4 + outputOffset] = (intValue1 >>> 16) & mask; + output[5 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[6 + outputOffset] = (intValue2 >>> 8) & mask; + output[7 + outputOffset] = intValue2 >>> 20; + output[8 + outputOffset] = intValue3 & mask; + output[9 + outputOffset] = (intValue3 >>> 12) & mask; + output[10 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[11 + outputOffset] = (intValue4 >>> 4) & mask; + output[12 + outputOffset] = (intValue4 >>> 16) & mask; + output[13 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[14 + outputOffset] = (intValue5 >>> 8) & mask; + output[15 + outputOffset] = intValue5 >>> 20; + output[16 + outputOffset] = intValue6 & mask; + output[17 + outputOffset] = (intValue6 >>> 12) & mask; + output[18 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[19 + outputOffset] = (intValue7 >>> 4) & mask; + output[20 + outputOffset] = (intValue7 >>> 16) & mask; + output[21 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[22 + outputOffset] = (intValue8 >>> 8) & mask; + output[23 + outputOffset] = intValue8 >>> 20; + output[24 + outputOffset] = intValue9 & mask; + output[25 + outputOffset] = (intValue9 >>> 12) & mask; + output[26 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[27 + outputOffset] = (intValue10 >>> 4) & mask; + output[28 + outputOffset] = (intValue10 >>> 16) & mask; + output[29 + outputOffset] = ((intValue10 >>> 28) | (intValue11 << 4)) & mask; + output[30 + outputOffset] = (intValue11 >>> 8) & mask; + output[31 + outputOffset] = intValue11 >>> 20; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For13Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For13Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For13Decompress.java (revision 0) @@ -0,0 +1,85 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For13Decompress extends ForDecompress { + static final int numFrameBits = 13; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 13) & mask; + output[2 + outputOffset] = ((intValue0 >>> 26) | (intValue1 << 6)) & mask; + output[3 + outputOffset] = (intValue1 >>> 7) & mask; + output[4 + outputOffset] = ((intValue1 >>> 20) | (intValue2 << 12)) & mask; + output[5 + outputOffset] = (intValue2 >>> 1) & mask; + output[6 + outputOffset] = (intValue2 >>> 14) & mask; + output[7 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask; + output[8 + outputOffset] = (intValue3 >>> 8) & mask; + output[9 + outputOffset] = ((intValue3 >>> 21) | (intValue4 << 11)) & mask; + output[10 + outputOffset] = (intValue4 >>> 2) & mask; + output[11 + outputOffset] = (intValue4 >>> 15) & mask; + output[12 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[13 + outputOffset] = (intValue5 >>> 9) & mask; + output[14 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; + output[15 + outputOffset] = (intValue6 >>> 3) & mask; + output[16 + outputOffset] = (intValue6 >>> 16) & mask; + output[17 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask; + output[18 + outputOffset] = (intValue7 >>> 10) & mask; + output[19 + outputOffset] = ((intValue7 >>> 23) | (intValue8 << 9)) & mask; + output[20 + outputOffset] = (intValue8 >>> 4) & mask; + output[21 + outputOffset] = (intValue8 >>> 17) & mask; + output[22 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[23 + outputOffset] = (intValue9 >>> 11) & mask; + output[24 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[25 + outputOffset] = (intValue10 >>> 5) & mask; + output[26 + outputOffset] = (intValue10 >>> 18) & mask; + output[27 + outputOffset] = ((intValue10 >>> 31) | (intValue11 << 1)) & mask; + output[28 + outputOffset] = (intValue11 >>> 12) & mask; + output[29 + outputOffset] = ((intValue11 >>> 25) | (intValue12 << 7)) & mask; + output[30 + outputOffset] = (intValue12 >>> 6) & mask; + output[31 + outputOffset] = intValue12 >>> 19; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For14Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For14Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For14Decompress.java (revision 0) @@ -0,0 +1,86 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For14Decompress extends ForDecompress { + static final int numFrameBits = 14; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 14) & mask; + output[2 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask; + output[3 + outputOffset] = (intValue1 >>> 10) & mask; + output[4 + outputOffset] = ((intValue1 >>> 24) | (intValue2 << 8)) & mask; + output[5 + outputOffset] = (intValue2 >>> 6) & mask; + output[6 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; + output[7 + outputOffset] = (intValue3 >>> 2) & mask; + output[8 + outputOffset] = (intValue3 >>> 16) & mask; + output[9 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[10 + outputOffset] = (intValue4 >>> 12) & mask; + output[11 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[12 + outputOffset] = (intValue5 >>> 8) & mask; + output[13 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; + output[14 + outputOffset] = (intValue6 >>> 4) & mask; + output[15 + outputOffset] = intValue6 >>> 18; + output[16 + outputOffset] = intValue7 & mask; + output[17 + outputOffset] = (intValue7 >>> 14) & mask; + output[18 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[19 + outputOffset] = (intValue8 >>> 10) & mask; + output[20 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[21 + outputOffset] = (intValue9 >>> 6) & mask; + output[22 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; + output[23 + outputOffset] = (intValue10 >>> 2) & mask; + output[24 + outputOffset] = (intValue10 >>> 16) & mask; + output[25 + outputOffset] = ((intValue10 >>> 30) | (intValue11 << 2)) & mask; + output[26 + outputOffset] = (intValue11 >>> 12) & mask; + output[27 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[28 + outputOffset] = (intValue12 >>> 8) & mask; + output[29 + outputOffset] = ((intValue12 >>> 22) | (intValue13 << 10)) & mask; + output[30 + outputOffset] = (intValue13 >>> 4) & mask; + output[31 + outputOffset] = intValue13 >>> 18; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For15Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For15Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For15Decompress.java (revision 0) @@ -0,0 +1,87 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For15Decompress extends ForDecompress { + static final int numFrameBits = 15; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 15) & mask; + output[2 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[3 + outputOffset] = (intValue1 >>> 13) & mask; + output[4 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[5 + outputOffset] = (intValue2 >>> 11) & mask; + output[6 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[7 + outputOffset] = (intValue3 >>> 9) & mask; + output[8 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[9 + outputOffset] = (intValue4 >>> 7) & mask; + output[10 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; + output[11 + outputOffset] = (intValue5 >>> 5) & mask; + output[12 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[13 + outputOffset] = (intValue6 >>> 3) & mask; + output[14 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; + output[15 + outputOffset] = (intValue7 >>> 1) & mask; + output[16 + outputOffset] = (intValue7 >>> 16) & mask; + output[17 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask; + output[18 + outputOffset] = (intValue8 >>> 14) & mask; + output[19 + outputOffset] = ((intValue8 >>> 29) | (intValue9 << 3)) & mask; + output[20 + outputOffset] = (intValue9 >>> 12) & mask; + output[21 + outputOffset] = ((intValue9 >>> 27) | (intValue10 << 5)) & mask; + output[22 + outputOffset] = (intValue10 >>> 10) & mask; + output[23 + outputOffset] = ((intValue10 >>> 25) | (intValue11 << 7)) & mask; + output[24 + outputOffset] = (intValue11 >>> 8) & mask; + output[25 + outputOffset] = ((intValue11 >>> 23) | (intValue12 << 9)) & mask; + output[26 + outputOffset] = (intValue12 >>> 6) & mask; + output[27 + outputOffset] = ((intValue12 >>> 21) | (intValue13 << 11)) & mask; + output[28 + outputOffset] = (intValue13 >>> 4) & mask; + output[29 + outputOffset] = ((intValue13 >>> 19) | (intValue14 << 13)) & mask; + output[30 + outputOffset] = (intValue14 >>> 2) & mask; + output[31 + outputOffset] = intValue14 >>> 17; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For16Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For16Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For16Decompress.java (revision 0) @@ -0,0 +1,88 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For16Decompress extends ForDecompress { + static final int numFrameBits = 16; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = intValue0 >>> 16; + output[2 + outputOffset] = intValue1 & mask; + output[3 + outputOffset] = intValue1 >>> 16; + output[4 + outputOffset] = intValue2 & mask; + output[5 + outputOffset] = intValue2 >>> 16; + output[6 + outputOffset] = intValue3 & mask; + output[7 + outputOffset] = intValue3 >>> 16; + output[8 + outputOffset] = intValue4 & mask; + output[9 + outputOffset] = intValue4 >>> 16; + output[10 + outputOffset] = intValue5 & mask; + output[11 + outputOffset] = intValue5 >>> 16; + output[12 + outputOffset] = intValue6 & mask; + output[13 + outputOffset] = intValue6 >>> 16; + output[14 + outputOffset] = intValue7 & mask; + output[15 + outputOffset] = intValue7 >>> 16; + output[16 + outputOffset] = intValue8 & mask; + output[17 + outputOffset] = intValue8 >>> 16; + output[18 + outputOffset] = intValue9 & mask; + output[19 + outputOffset] = intValue9 >>> 16; + output[20 + outputOffset] = intValue10 & mask; + output[21 + outputOffset] = intValue10 >>> 16; + output[22 + outputOffset] = intValue11 & mask; + output[23 + outputOffset] = intValue11 >>> 16; + output[24 + outputOffset] = intValue12 & mask; + output[25 + outputOffset] = intValue12 >>> 16; + output[26 + outputOffset] = intValue13 & mask; + output[27 + outputOffset] = intValue13 >>> 16; + output[28 + outputOffset] = intValue14 & mask; + output[29 + outputOffset] = intValue14 >>> 16; + output[30 + outputOffset] = intValue15 & mask; + output[31 + outputOffset] = intValue15 >>> 16; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For17Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For17Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For17Decompress.java (revision 0) @@ -0,0 +1,89 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For17Decompress extends ForDecompress { + static final int numFrameBits = 17; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 17) | (intValue1 << 15)) & mask; + output[2 + outputOffset] = (intValue1 >>> 2) & mask; + output[3 + outputOffset] = ((intValue1 >>> 19) | (intValue2 << 13)) & mask; + output[4 + outputOffset] = (intValue2 >>> 4) & mask; + output[5 + outputOffset] = ((intValue2 >>> 21) | (intValue3 << 11)) & mask; + output[6 + outputOffset] = (intValue3 >>> 6) & mask; + output[7 + outputOffset] = ((intValue3 >>> 23) | (intValue4 << 9)) & mask; + output[8 + outputOffset] = (intValue4 >>> 8) & mask; + output[9 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask; + output[10 + outputOffset] = (intValue5 >>> 10) & mask; + output[11 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask; + output[12 + outputOffset] = (intValue6 >>> 12) & mask; + output[13 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask; + output[14 + outputOffset] = (intValue7 >>> 14) & mask; + output[15 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask; + output[16 + outputOffset] = ((intValue8 >>> 16) | (intValue9 << 16)) & mask; + output[17 + outputOffset] = (intValue9 >>> 1) & mask; + output[18 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask; + output[19 + outputOffset] = (intValue10 >>> 3) & mask; + output[20 + outputOffset] = ((intValue10 >>> 20) | (intValue11 << 12)) & mask; + output[21 + outputOffset] = (intValue11 >>> 5) & mask; + output[22 + outputOffset] = ((intValue11 >>> 22) | (intValue12 << 10)) & mask; + output[23 + outputOffset] = (intValue12 >>> 7) & mask; + output[24 + outputOffset] = ((intValue12 >>> 24) | (intValue13 << 8)) & mask; + output[25 + outputOffset] = (intValue13 >>> 9) & mask; + output[26 + outputOffset] = ((intValue13 >>> 26) | (intValue14 << 6)) & mask; + output[27 + outputOffset] = (intValue14 >>> 11) & mask; + output[28 + outputOffset] = ((intValue14 >>> 28) | (intValue15 << 4)) & mask; + output[29 + outputOffset] = (intValue15 >>> 13) & mask; + output[30 + outputOffset] = ((intValue15 >>> 30) | (intValue16 << 2)) & mask; + output[31 + outputOffset] = intValue16 >>> 15; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For18Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For18Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For18Decompress.java (revision 0) @@ -0,0 +1,90 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For18Decompress extends ForDecompress { + static final int numFrameBits = 18; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 18) | (intValue1 << 14)) & mask; + output[2 + outputOffset] = (intValue1 >>> 4) & mask; + output[3 + outputOffset] = ((intValue1 >>> 22) | (intValue2 << 10)) & mask; + output[4 + outputOffset] = (intValue2 >>> 8) & mask; + output[5 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[6 + outputOffset] = (intValue3 >>> 12) & mask; + output[7 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[8 + outputOffset] = ((intValue4 >>> 16) | (intValue5 << 16)) & mask; + output[9 + outputOffset] = (intValue5 >>> 2) & mask; + output[10 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[11 + outputOffset] = (intValue6 >>> 6) & mask; + output[12 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[13 + outputOffset] = (intValue7 >>> 10) & mask; + output[14 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[15 + outputOffset] = intValue8 >>> 14; + output[16 + outputOffset] = intValue9 & mask; + output[17 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask; + output[18 + outputOffset] = (intValue10 >>> 4) & mask; + output[19 + outputOffset] = ((intValue10 >>> 22) | (intValue11 << 10)) & mask; + output[20 + outputOffset] = (intValue11 >>> 8) & mask; + output[21 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[22 + outputOffset] = (intValue12 >>> 12) & mask; + output[23 + outputOffset] = ((intValue12 >>> 30) | (intValue13 << 2)) & mask; + output[24 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; + output[25 + outputOffset] = (intValue14 >>> 2) & mask; + output[26 + outputOffset] = ((intValue14 >>> 20) | (intValue15 << 12)) & mask; + output[27 + outputOffset] = (intValue15 >>> 6) & mask; + output[28 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[29 + outputOffset] = (intValue16 >>> 10) & mask; + output[30 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[31 + outputOffset] = intValue17 >>> 14; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For19Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For19Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For19Decompress.java (revision 0) @@ -0,0 +1,91 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For19Decompress extends ForDecompress { + static final int numFrameBits = 19; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 19) | (intValue1 << 13)) & mask; + output[2 + outputOffset] = (intValue1 >>> 6) & mask; + output[3 + outputOffset] = ((intValue1 >>> 25) | (intValue2 << 7)) & mask; + output[4 + outputOffset] = (intValue2 >>> 12) & mask; + output[5 + outputOffset] = ((intValue2 >>> 31) | (intValue3 << 1)) & mask; + output[6 + outputOffset] = ((intValue3 >>> 18) | (intValue4 << 14)) & mask; + output[7 + outputOffset] = (intValue4 >>> 5) & mask; + output[8 + outputOffset] = ((intValue4 >>> 24) | (intValue5 << 8)) & mask; + output[9 + outputOffset] = (intValue5 >>> 11) & mask; + output[10 + outputOffset] = ((intValue5 >>> 30) | (intValue6 << 2)) & mask; + output[11 + outputOffset] = ((intValue6 >>> 17) | (intValue7 << 15)) & mask; + output[12 + outputOffset] = (intValue7 >>> 4) & mask; + output[13 + outputOffset] = ((intValue7 >>> 23) | (intValue8 << 9)) & mask; + output[14 + outputOffset] = (intValue8 >>> 10) & mask; + output[15 + outputOffset] = ((intValue8 >>> 29) | (intValue9 << 3)) & mask; + output[16 + outputOffset] = ((intValue9 >>> 16) | (intValue10 << 16)) & mask; + output[17 + outputOffset] = (intValue10 >>> 3) & mask; + output[18 + outputOffset] = ((intValue10 >>> 22) | (intValue11 << 10)) & mask; + output[19 + outputOffset] = (intValue11 >>> 9) & mask; + output[20 + outputOffset] = ((intValue11 >>> 28) | (intValue12 << 4)) & mask; + output[21 + outputOffset] = ((intValue12 >>> 15) | (intValue13 << 17)) & mask; + output[22 + outputOffset] = (intValue13 >>> 2) & mask; + output[23 + outputOffset] = ((intValue13 >>> 21) | (intValue14 << 11)) & mask; + output[24 + outputOffset] = (intValue14 >>> 8) & mask; + output[25 + outputOffset] = ((intValue14 >>> 27) | (intValue15 << 5)) & mask; + output[26 + outputOffset] = ((intValue15 >>> 14) | (intValue16 << 18)) & mask; + output[27 + outputOffset] = (intValue16 >>> 1) & mask; + output[28 + outputOffset] = ((intValue16 >>> 20) | (intValue17 << 12)) & mask; + output[29 + outputOffset] = (intValue17 >>> 7) & mask; + output[30 + outputOffset] = ((intValue17 >>> 26) | (intValue18 << 6)) & mask; + output[31 + outputOffset] = intValue18 >>> 13; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For1Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For1Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For1Decompress.java (revision 0) @@ -0,0 +1,73 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For1Decompress extends ForDecompress { + static final int numFrameBits = 1; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 1) & mask; + output[2 + outputOffset] = (intValue0 >>> 2) & mask; + output[3 + outputOffset] = (intValue0 >>> 3) & mask; + output[4 + outputOffset] = (intValue0 >>> 4) & mask; + output[5 + outputOffset] = (intValue0 >>> 5) & mask; + output[6 + outputOffset] = (intValue0 >>> 6) & mask; + output[7 + outputOffset] = (intValue0 >>> 7) & mask; + output[8 + outputOffset] = (intValue0 >>> 8) & mask; + output[9 + outputOffset] = (intValue0 >>> 9) & mask; + output[10 + outputOffset] = (intValue0 >>> 10) & mask; + output[11 + outputOffset] = (intValue0 >>> 11) & mask; + output[12 + outputOffset] = (intValue0 >>> 12) & mask; + output[13 + outputOffset] = (intValue0 >>> 13) & mask; + output[14 + outputOffset] = (intValue0 >>> 14) & mask; + output[15 + outputOffset] = (intValue0 >>> 15) & mask; + output[16 + outputOffset] = (intValue0 >>> 16) & mask; + output[17 + outputOffset] = (intValue0 >>> 17) & mask; + output[18 + outputOffset] = (intValue0 >>> 18) & mask; + output[19 + outputOffset] = (intValue0 >>> 19) & mask; + output[20 + outputOffset] = (intValue0 >>> 20) & mask; + output[21 + outputOffset] = (intValue0 >>> 21) & mask; + output[22 + outputOffset] = (intValue0 >>> 22) & mask; + output[23 + outputOffset] = (intValue0 >>> 23) & mask; + output[24 + outputOffset] = (intValue0 >>> 24) & mask; + output[25 + outputOffset] = (intValue0 >>> 25) & mask; + output[26 + outputOffset] = (intValue0 >>> 26) & mask; + output[27 + outputOffset] = (intValue0 >>> 27) & mask; + output[28 + outputOffset] = (intValue0 >>> 28) & mask; + output[29 + outputOffset] = (intValue0 >>> 29) & mask; + output[30 + outputOffset] = (intValue0 >>> 30) & mask; + output[31 + outputOffset] = intValue0 >>> 31; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For20Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For20Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For20Decompress.java (revision 0) @@ -0,0 +1,92 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For20Decompress extends ForDecompress { + static final int numFrameBits = 20; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 20) | (intValue1 << 12)) & mask; + output[2 + outputOffset] = (intValue1 >>> 8) & mask; + output[3 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[4 + outputOffset] = ((intValue2 >>> 16) | (intValue3 << 16)) & mask; + output[5 + outputOffset] = (intValue3 >>> 4) & mask; + output[6 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[7 + outputOffset] = intValue4 >>> 12; + output[8 + outputOffset] = intValue5 & mask; + output[9 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[10 + outputOffset] = (intValue6 >>> 8) & mask; + output[11 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[12 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; + output[13 + outputOffset] = (intValue8 >>> 4) & mask; + output[14 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[15 + outputOffset] = intValue9 >>> 12; + output[16 + outputOffset] = intValue10 & mask; + output[17 + outputOffset] = ((intValue10 >>> 20) | (intValue11 << 12)) & mask; + output[18 + outputOffset] = (intValue11 >>> 8) & mask; + output[19 + outputOffset] = ((intValue11 >>> 28) | (intValue12 << 4)) & mask; + output[20 + outputOffset] = ((intValue12 >>> 16) | (intValue13 << 16)) & mask; + output[21 + outputOffset] = (intValue13 >>> 4) & mask; + output[22 + outputOffset] = ((intValue13 >>> 24) | (intValue14 << 8)) & mask; + output[23 + outputOffset] = intValue14 >>> 12; + output[24 + outputOffset] = intValue15 & mask; + output[25 + outputOffset] = ((intValue15 >>> 20) | (intValue16 << 12)) & mask; + output[26 + outputOffset] = (intValue16 >>> 8) & mask; + output[27 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[28 + outputOffset] = ((intValue17 >>> 16) | (intValue18 << 16)) & mask; + output[29 + outputOffset] = (intValue18 >>> 4) & mask; + output[30 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[31 + outputOffset] = intValue19 >>> 12; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For21Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For21Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For21Decompress.java (revision 0) @@ -0,0 +1,93 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For21Decompress extends ForDecompress { + static final int numFrameBits = 21; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 21) | (intValue1 << 11)) & mask; + output[2 + outputOffset] = (intValue1 >>> 10) & mask; + output[3 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[4 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; + output[5 + outputOffset] = (intValue3 >>> 9) & mask; + output[6 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[7 + outputOffset] = ((intValue4 >>> 19) | (intValue5 << 13)) & mask; + output[8 + outputOffset] = (intValue5 >>> 8) & mask; + output[9 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[10 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; + output[11 + outputOffset] = (intValue7 >>> 7) & mask; + output[12 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[13 + outputOffset] = ((intValue8 >>> 17) | (intValue9 << 15)) & mask; + output[14 + outputOffset] = (intValue9 >>> 6) & mask; + output[15 + outputOffset] = ((intValue9 >>> 27) | (intValue10 << 5)) & mask; + output[16 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; + output[17 + outputOffset] = (intValue11 >>> 5) & mask; + output[18 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[19 + outputOffset] = ((intValue12 >>> 15) | (intValue13 << 17)) & mask; + output[20 + outputOffset] = (intValue13 >>> 4) & mask; + output[21 + outputOffset] = ((intValue13 >>> 25) | (intValue14 << 7)) & mask; + output[22 + outputOffset] = ((intValue14 >>> 14) | (intValue15 << 18)) & mask; + output[23 + outputOffset] = (intValue15 >>> 3) & mask; + output[24 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[25 + outputOffset] = ((intValue16 >>> 13) | (intValue17 << 19)) & mask; + output[26 + outputOffset] = (intValue17 >>> 2) & mask; + output[27 + outputOffset] = ((intValue17 >>> 23) | (intValue18 << 9)) & mask; + output[28 + outputOffset] = ((intValue18 >>> 12) | (intValue19 << 20)) & mask; + output[29 + outputOffset] = (intValue19 >>> 1) & mask; + output[30 + outputOffset] = ((intValue19 >>> 22) | (intValue20 << 10)) & mask; + output[31 + outputOffset] = intValue20 >>> 11; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For22Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For22Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For22Decompress.java (revision 0) @@ -0,0 +1,94 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For22Decompress extends ForDecompress { + static final int numFrameBits = 22; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 22) | (intValue1 << 10)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 12) | (intValue2 << 20)) & mask; + output[3 + outputOffset] = (intValue2 >>> 2) & mask; + output[4 + outputOffset] = ((intValue2 >>> 24) | (intValue3 << 8)) & mask; + output[5 + outputOffset] = ((intValue3 >>> 14) | (intValue4 << 18)) & mask; + output[6 + outputOffset] = (intValue4 >>> 4) & mask; + output[7 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[8 + outputOffset] = ((intValue5 >>> 16) | (intValue6 << 16)) & mask; + output[9 + outputOffset] = (intValue6 >>> 6) & mask; + output[10 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[11 + outputOffset] = ((intValue7 >>> 18) | (intValue8 << 14)) & mask; + output[12 + outputOffset] = (intValue8 >>> 8) & mask; + output[13 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[14 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; + output[15 + outputOffset] = intValue10 >>> 10; + output[16 + outputOffset] = intValue11 & mask; + output[17 + outputOffset] = ((intValue11 >>> 22) | (intValue12 << 10)) & mask; + output[18 + outputOffset] = ((intValue12 >>> 12) | (intValue13 << 20)) & mask; + output[19 + outputOffset] = (intValue13 >>> 2) & mask; + output[20 + outputOffset] = ((intValue13 >>> 24) | (intValue14 << 8)) & mask; + output[21 + outputOffset] = ((intValue14 >>> 14) | (intValue15 << 18)) & mask; + output[22 + outputOffset] = (intValue15 >>> 4) & mask; + output[23 + outputOffset] = ((intValue15 >>> 26) | (intValue16 << 6)) & mask; + output[24 + outputOffset] = ((intValue16 >>> 16) | (intValue17 << 16)) & mask; + output[25 + outputOffset] = (intValue17 >>> 6) & mask; + output[26 + outputOffset] = ((intValue17 >>> 28) | (intValue18 << 4)) & mask; + output[27 + outputOffset] = ((intValue18 >>> 18) | (intValue19 << 14)) & mask; + output[28 + outputOffset] = (intValue19 >>> 8) & mask; + output[29 + outputOffset] = ((intValue19 >>> 30) | (intValue20 << 2)) & mask; + output[30 + outputOffset] = ((intValue20 >>> 20) | (intValue21 << 12)) & mask; + output[31 + outputOffset] = intValue21 >>> 10; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For23Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For23Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For23Decompress.java (revision 0) @@ -0,0 +1,95 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For23Decompress extends ForDecompress { + static final int numFrameBits = 23; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 23) | (intValue1 << 9)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 14) | (intValue2 << 18)) & mask; + output[3 + outputOffset] = (intValue2 >>> 5) & mask; + output[4 + outputOffset] = ((intValue2 >>> 28) | (intValue3 << 4)) & mask; + output[5 + outputOffset] = ((intValue3 >>> 19) | (intValue4 << 13)) & mask; + output[6 + outputOffset] = ((intValue4 >>> 10) | (intValue5 << 22)) & mask; + output[7 + outputOffset] = (intValue5 >>> 1) & mask; + output[8 + outputOffset] = ((intValue5 >>> 24) | (intValue6 << 8)) & mask; + output[9 + outputOffset] = ((intValue6 >>> 15) | (intValue7 << 17)) & mask; + output[10 + outputOffset] = (intValue7 >>> 6) & mask; + output[11 + outputOffset] = ((intValue7 >>> 29) | (intValue8 << 3)) & mask; + output[12 + outputOffset] = ((intValue8 >>> 20) | (intValue9 << 12)) & mask; + output[13 + outputOffset] = ((intValue9 >>> 11) | (intValue10 << 21)) & mask; + output[14 + outputOffset] = (intValue10 >>> 2) & mask; + output[15 + outputOffset] = ((intValue10 >>> 25) | (intValue11 << 7)) & mask; + output[16 + outputOffset] = ((intValue11 >>> 16) | (intValue12 << 16)) & mask; + output[17 + outputOffset] = (intValue12 >>> 7) & mask; + output[18 + outputOffset] = ((intValue12 >>> 30) | (intValue13 << 2)) & mask; + output[19 + outputOffset] = ((intValue13 >>> 21) | (intValue14 << 11)) & mask; + output[20 + outputOffset] = ((intValue14 >>> 12) | (intValue15 << 20)) & mask; + output[21 + outputOffset] = (intValue15 >>> 3) & mask; + output[22 + outputOffset] = ((intValue15 >>> 26) | (intValue16 << 6)) & mask; + output[23 + outputOffset] = ((intValue16 >>> 17) | (intValue17 << 15)) & mask; + output[24 + outputOffset] = (intValue17 >>> 8) & mask; + output[25 + outputOffset] = ((intValue17 >>> 31) | (intValue18 << 1)) & mask; + output[26 + outputOffset] = ((intValue18 >>> 22) | (intValue19 << 10)) & mask; + output[27 + outputOffset] = ((intValue19 >>> 13) | (intValue20 << 19)) & mask; + output[28 + outputOffset] = (intValue20 >>> 4) & mask; + output[29 + outputOffset] = ((intValue20 >>> 27) | (intValue21 << 5)) & mask; + output[30 + outputOffset] = ((intValue21 >>> 18) | (intValue22 << 14)) & mask; + output[31 + outputOffset] = intValue22 >>> 9; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For24Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For24Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For24Decompress.java (revision 0) @@ -0,0 +1,96 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For24Decompress extends ForDecompress { + static final int numFrameBits = 24; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 24) | (intValue1 << 8)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 16) | (intValue2 << 16)) & mask; + output[3 + outputOffset] = intValue2 >>> 8; + output[4 + outputOffset] = intValue3 & mask; + output[5 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[6 + outputOffset] = ((intValue4 >>> 16) | (intValue5 << 16)) & mask; + output[7 + outputOffset] = intValue5 >>> 8; + output[8 + outputOffset] = intValue6 & mask; + output[9 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[10 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; + output[11 + outputOffset] = intValue8 >>> 8; + output[12 + outputOffset] = intValue9 & mask; + output[13 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[14 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; + output[15 + outputOffset] = intValue11 >>> 8; + output[16 + outputOffset] = intValue12 & mask; + output[17 + outputOffset] = ((intValue12 >>> 24) | (intValue13 << 8)) & mask; + output[18 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; + output[19 + outputOffset] = intValue14 >>> 8; + output[20 + outputOffset] = intValue15 & mask; + output[21 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[22 + outputOffset] = ((intValue16 >>> 16) | (intValue17 << 16)) & mask; + output[23 + outputOffset] = intValue17 >>> 8; + output[24 + outputOffset] = intValue18 & mask; + output[25 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[26 + outputOffset] = ((intValue19 >>> 16) | (intValue20 << 16)) & mask; + output[27 + outputOffset] = intValue20 >>> 8; + output[28 + outputOffset] = intValue21 & mask; + output[29 + outputOffset] = ((intValue21 >>> 24) | (intValue22 << 8)) & mask; + output[30 + outputOffset] = ((intValue22 >>> 16) | (intValue23 << 16)) & mask; + output[31 + outputOffset] = intValue23 >>> 8; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For25Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For25Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For25Decompress.java (revision 0) @@ -0,0 +1,97 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For25Decompress extends ForDecompress { + static final int numFrameBits = 25; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + int intValue24 = compressedBuffer.get(bufIndex + 24); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 25) | (intValue1 << 7)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 18) | (intValue2 << 14)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 11) | (intValue3 << 21)) & mask; + output[4 + outputOffset] = (intValue3 >>> 4) & mask; + output[5 + outputOffset] = ((intValue3 >>> 29) | (intValue4 << 3)) & mask; + output[6 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; + output[7 + outputOffset] = ((intValue5 >>> 15) | (intValue6 << 17)) & mask; + output[8 + outputOffset] = ((intValue6 >>> 8) | (intValue7 << 24)) & mask; + output[9 + outputOffset] = (intValue7 >>> 1) & mask; + output[10 + outputOffset] = ((intValue7 >>> 26) | (intValue8 << 6)) & mask; + output[11 + outputOffset] = ((intValue8 >>> 19) | (intValue9 << 13)) & mask; + output[12 + outputOffset] = ((intValue9 >>> 12) | (intValue10 << 20)) & mask; + output[13 + outputOffset] = (intValue10 >>> 5) & mask; + output[14 + outputOffset] = ((intValue10 >>> 30) | (intValue11 << 2)) & mask; + output[15 + outputOffset] = ((intValue11 >>> 23) | (intValue12 << 9)) & mask; + output[16 + outputOffset] = ((intValue12 >>> 16) | (intValue13 << 16)) & mask; + output[17 + outputOffset] = ((intValue13 >>> 9) | (intValue14 << 23)) & mask; + output[18 + outputOffset] = (intValue14 >>> 2) & mask; + output[19 + outputOffset] = ((intValue14 >>> 27) | (intValue15 << 5)) & mask; + output[20 + outputOffset] = ((intValue15 >>> 20) | (intValue16 << 12)) & mask; + output[21 + outputOffset] = ((intValue16 >>> 13) | (intValue17 << 19)) & mask; + output[22 + outputOffset] = (intValue17 >>> 6) & mask; + output[23 + outputOffset] = ((intValue17 >>> 31) | (intValue18 << 1)) & mask; + output[24 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[25 + outputOffset] = ((intValue19 >>> 17) | (intValue20 << 15)) & mask; + output[26 + outputOffset] = ((intValue20 >>> 10) | (intValue21 << 22)) & mask; + output[27 + outputOffset] = (intValue21 >>> 3) & mask; + output[28 + outputOffset] = ((intValue21 >>> 28) | (intValue22 << 4)) & mask; + output[29 + outputOffset] = ((intValue22 >>> 21) | (intValue23 << 11)) & mask; + output[30 + outputOffset] = ((intValue23 >>> 14) | (intValue24 << 18)) & mask; + output[31 + outputOffset] = intValue24 >>> 7; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For26Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For26Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For26Decompress.java (revision 0) @@ -0,0 +1,98 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For26Decompress extends ForDecompress { + static final int numFrameBits = 26; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + int intValue24 = compressedBuffer.get(bufIndex + 24); + int intValue25 = compressedBuffer.get(bufIndex + 25); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 26) | (intValue1 << 6)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 20) | (intValue2 << 12)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 14) | (intValue3 << 18)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 8) | (intValue4 << 24)) & mask; + output[5 + outputOffset] = (intValue4 >>> 2) & mask; + output[6 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[7 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; + output[8 + outputOffset] = ((intValue6 >>> 16) | (intValue7 << 16)) & mask; + output[9 + outputOffset] = ((intValue7 >>> 10) | (intValue8 << 22)) & mask; + output[10 + outputOffset] = (intValue8 >>> 4) & mask; + output[11 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[12 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[13 + outputOffset] = ((intValue10 >>> 18) | (intValue11 << 14)) & mask; + output[14 + outputOffset] = ((intValue11 >>> 12) | (intValue12 << 20)) & mask; + output[15 + outputOffset] = intValue12 >>> 6; + output[16 + outputOffset] = intValue13 & mask; + output[17 + outputOffset] = ((intValue13 >>> 26) | (intValue14 << 6)) & mask; + output[18 + outputOffset] = ((intValue14 >>> 20) | (intValue15 << 12)) & mask; + output[19 + outputOffset] = ((intValue15 >>> 14) | (intValue16 << 18)) & mask; + output[20 + outputOffset] = ((intValue16 >>> 8) | (intValue17 << 24)) & mask; + output[21 + outputOffset] = (intValue17 >>> 2) & mask; + output[22 + outputOffset] = ((intValue17 >>> 28) | (intValue18 << 4)) & mask; + output[23 + outputOffset] = ((intValue18 >>> 22) | (intValue19 << 10)) & mask; + output[24 + outputOffset] = ((intValue19 >>> 16) | (intValue20 << 16)) & mask; + output[25 + outputOffset] = ((intValue20 >>> 10) | (intValue21 << 22)) & mask; + output[26 + outputOffset] = (intValue21 >>> 4) & mask; + output[27 + outputOffset] = ((intValue21 >>> 30) | (intValue22 << 2)) & mask; + output[28 + outputOffset] = ((intValue22 >>> 24) | (intValue23 << 8)) & mask; + output[29 + outputOffset] = ((intValue23 >>> 18) | (intValue24 << 14)) & mask; + output[30 + outputOffset] = ((intValue24 >>> 12) | (intValue25 << 20)) & mask; + output[31 + outputOffset] = intValue25 >>> 6; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For27Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For27Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For27Decompress.java (revision 0) @@ -0,0 +1,99 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For27Decompress extends ForDecompress { + static final int numFrameBits = 27; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + int intValue24 = compressedBuffer.get(bufIndex + 24); + int intValue25 = compressedBuffer.get(bufIndex + 25); + int intValue26 = compressedBuffer.get(bufIndex + 26); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 27) | (intValue1 << 5)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 22) | (intValue2 << 10)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 17) | (intValue3 << 15)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 12) | (intValue4 << 20)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 7) | (intValue5 << 25)) & mask; + output[6 + outputOffset] = (intValue5 >>> 2) & mask; + output[7 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[8 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[9 + outputOffset] = ((intValue7 >>> 19) | (intValue8 << 13)) & mask; + output[10 + outputOffset] = ((intValue8 >>> 14) | (intValue9 << 18)) & mask; + output[11 + outputOffset] = ((intValue9 >>> 9) | (intValue10 << 23)) & mask; + output[12 + outputOffset] = (intValue10 >>> 4) & mask; + output[13 + outputOffset] = ((intValue10 >>> 31) | (intValue11 << 1)) & mask; + output[14 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[15 + outputOffset] = ((intValue12 >>> 21) | (intValue13 << 11)) & mask; + output[16 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; + output[17 + outputOffset] = ((intValue14 >>> 11) | (intValue15 << 21)) & mask; + output[18 + outputOffset] = ((intValue15 >>> 6) | (intValue16 << 26)) & mask; + output[19 + outputOffset] = (intValue16 >>> 1) & mask; + output[20 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[21 + outputOffset] = ((intValue17 >>> 23) | (intValue18 << 9)) & mask; + output[22 + outputOffset] = ((intValue18 >>> 18) | (intValue19 << 14)) & mask; + output[23 + outputOffset] = ((intValue19 >>> 13) | (intValue20 << 19)) & mask; + output[24 + outputOffset] = ((intValue20 >>> 8) | (intValue21 << 24)) & mask; + output[25 + outputOffset] = (intValue21 >>> 3) & mask; + output[26 + outputOffset] = ((intValue21 >>> 30) | (intValue22 << 2)) & mask; + output[27 + outputOffset] = ((intValue22 >>> 25) | (intValue23 << 7)) & mask; + output[28 + outputOffset] = ((intValue23 >>> 20) | (intValue24 << 12)) & mask; + output[29 + outputOffset] = ((intValue24 >>> 15) | (intValue25 << 17)) & mask; + output[30 + outputOffset] = ((intValue25 >>> 10) | (intValue26 << 22)) & mask; + output[31 + outputOffset] = intValue26 >>> 5; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For28Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For28Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For28Decompress.java (revision 0) @@ -0,0 +1,100 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For28Decompress extends ForDecompress { + static final int numFrameBits = 28; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + int intValue24 = compressedBuffer.get(bufIndex + 24); + int intValue25 = compressedBuffer.get(bufIndex + 25); + int intValue26 = compressedBuffer.get(bufIndex + 26); + int intValue27 = compressedBuffer.get(bufIndex + 27); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 24) | (intValue2 << 8)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 16) | (intValue4 << 16)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 12) | (intValue5 << 20)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 8) | (intValue6 << 24)) & mask; + output[7 + outputOffset] = intValue6 >>> 4; + output[8 + outputOffset] = intValue7 & mask; + output[9 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[10 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[11 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; + output[12 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; + output[13 + outputOffset] = ((intValue11 >>> 12) | (intValue12 << 20)) & mask; + output[14 + outputOffset] = ((intValue12 >>> 8) | (intValue13 << 24)) & mask; + output[15 + outputOffset] = intValue13 >>> 4; + output[16 + outputOffset] = intValue14 & mask; + output[17 + outputOffset] = ((intValue14 >>> 28) | (intValue15 << 4)) & mask; + output[18 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[19 + outputOffset] = ((intValue16 >>> 20) | (intValue17 << 12)) & mask; + output[20 + outputOffset] = ((intValue17 >>> 16) | (intValue18 << 16)) & mask; + output[21 + outputOffset] = ((intValue18 >>> 12) | (intValue19 << 20)) & mask; + output[22 + outputOffset] = ((intValue19 >>> 8) | (intValue20 << 24)) & mask; + output[23 + outputOffset] = intValue20 >>> 4; + output[24 + outputOffset] = intValue21 & mask; + output[25 + outputOffset] = ((intValue21 >>> 28) | (intValue22 << 4)) & mask; + output[26 + outputOffset] = ((intValue22 >>> 24) | (intValue23 << 8)) & mask; + output[27 + outputOffset] = ((intValue23 >>> 20) | (intValue24 << 12)) & mask; + output[28 + outputOffset] = ((intValue24 >>> 16) | (intValue25 << 16)) & mask; + output[29 + outputOffset] = ((intValue25 >>> 12) | (intValue26 << 20)) & mask; + output[30 + outputOffset] = ((intValue26 >>> 8) | (intValue27 << 24)) & mask; + output[31 + outputOffset] = intValue27 >>> 4; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For29Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For29Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For29Decompress.java (revision 0) @@ -0,0 +1,101 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For29Decompress extends ForDecompress { + static final int numFrameBits = 29; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + int intValue24 = compressedBuffer.get(bufIndex + 24); + int intValue25 = compressedBuffer.get(bufIndex + 25); + int intValue26 = compressedBuffer.get(bufIndex + 26); + int intValue27 = compressedBuffer.get(bufIndex + 27); + int intValue28 = compressedBuffer.get(bufIndex + 28); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 29) | (intValue1 << 3)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 26) | (intValue2 << 6)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 23) | (intValue3 << 9)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 20) | (intValue4 << 12)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 17) | (intValue5 << 15)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 14) | (intValue6 << 18)) & mask; + output[7 + outputOffset] = ((intValue6 >>> 11) | (intValue7 << 21)) & mask; + output[8 + outputOffset] = ((intValue7 >>> 8) | (intValue8 << 24)) & mask; + output[9 + outputOffset] = ((intValue8 >>> 5) | (intValue9 << 27)) & mask; + output[10 + outputOffset] = (intValue9 >>> 2) & mask; + output[11 + outputOffset] = ((intValue9 >>> 31) | (intValue10 << 1)) & mask; + output[12 + outputOffset] = ((intValue10 >>> 28) | (intValue11 << 4)) & mask; + output[13 + outputOffset] = ((intValue11 >>> 25) | (intValue12 << 7)) & mask; + output[14 + outputOffset] = ((intValue12 >>> 22) | (intValue13 << 10)) & mask; + output[15 + outputOffset] = ((intValue13 >>> 19) | (intValue14 << 13)) & mask; + output[16 + outputOffset] = ((intValue14 >>> 16) | (intValue15 << 16)) & mask; + output[17 + outputOffset] = ((intValue15 >>> 13) | (intValue16 << 19)) & mask; + output[18 + outputOffset] = ((intValue16 >>> 10) | (intValue17 << 22)) & mask; + output[19 + outputOffset] = ((intValue17 >>> 7) | (intValue18 << 25)) & mask; + output[20 + outputOffset] = ((intValue18 >>> 4) | (intValue19 << 28)) & mask; + output[21 + outputOffset] = (intValue19 >>> 1) & mask; + output[22 + outputOffset] = ((intValue19 >>> 30) | (intValue20 << 2)) & mask; + output[23 + outputOffset] = ((intValue20 >>> 27) | (intValue21 << 5)) & mask; + output[24 + outputOffset] = ((intValue21 >>> 24) | (intValue22 << 8)) & mask; + output[25 + outputOffset] = ((intValue22 >>> 21) | (intValue23 << 11)) & mask; + output[26 + outputOffset] = ((intValue23 >>> 18) | (intValue24 << 14)) & mask; + output[27 + outputOffset] = ((intValue24 >>> 15) | (intValue25 << 17)) & mask; + output[28 + outputOffset] = ((intValue25 >>> 12) | (intValue26 << 20)) & mask; + output[29 + outputOffset] = ((intValue26 >>> 9) | (intValue27 << 23)) & mask; + output[30 + outputOffset] = ((intValue27 >>> 6) | (intValue28 << 26)) & mask; + output[31 + outputOffset] = intValue28 >>> 3; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For2Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For2Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For2Decompress.java (revision 0) @@ -0,0 +1,74 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For2Decompress extends ForDecompress { + static final int numFrameBits = 2; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 2) & mask; + output[2 + outputOffset] = (intValue0 >>> 4) & mask; + output[3 + outputOffset] = (intValue0 >>> 6) & mask; + output[4 + outputOffset] = (intValue0 >>> 8) & mask; + output[5 + outputOffset] = (intValue0 >>> 10) & mask; + output[6 + outputOffset] = (intValue0 >>> 12) & mask; + output[7 + outputOffset] = (intValue0 >>> 14) & mask; + output[8 + outputOffset] = (intValue0 >>> 16) & mask; + output[9 + outputOffset] = (intValue0 >>> 18) & mask; + output[10 + outputOffset] = (intValue0 >>> 20) & mask; + output[11 + outputOffset] = (intValue0 >>> 22) & mask; + output[12 + outputOffset] = (intValue0 >>> 24) & mask; + output[13 + outputOffset] = (intValue0 >>> 26) & mask; + output[14 + outputOffset] = (intValue0 >>> 28) & mask; + output[15 + outputOffset] = intValue0 >>> 30; + output[16 + outputOffset] = intValue1 & mask; + output[17 + outputOffset] = (intValue1 >>> 2) & mask; + output[18 + outputOffset] = (intValue1 >>> 4) & mask; + output[19 + outputOffset] = (intValue1 >>> 6) & mask; + output[20 + outputOffset] = (intValue1 >>> 8) & mask; + output[21 + outputOffset] = (intValue1 >>> 10) & mask; + output[22 + outputOffset] = (intValue1 >>> 12) & mask; + output[23 + outputOffset] = (intValue1 >>> 14) & mask; + output[24 + outputOffset] = (intValue1 >>> 16) & mask; + output[25 + outputOffset] = (intValue1 >>> 18) & mask; + output[26 + outputOffset] = (intValue1 >>> 20) & mask; + output[27 + outputOffset] = (intValue1 >>> 22) & mask; + output[28 + outputOffset] = (intValue1 >>> 24) & mask; + output[29 + outputOffset] = (intValue1 >>> 26) & mask; + output[30 + outputOffset] = (intValue1 >>> 28) & mask; + output[31 + outputOffset] = intValue1 >>> 30; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For30Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For30Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For30Decompress.java (revision 0) @@ -0,0 +1,102 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For30Decompress extends ForDecompress { + static final int numFrameBits = 30; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + int intValue24 = compressedBuffer.get(bufIndex + 24); + int intValue25 = compressedBuffer.get(bufIndex + 25); + int intValue26 = compressedBuffer.get(bufIndex + 26); + int intValue27 = compressedBuffer.get(bufIndex + 27); + int intValue28 = compressedBuffer.get(bufIndex + 28); + int intValue29 = compressedBuffer.get(bufIndex + 29); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[7 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; + output[8 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; + output[9 + outputOffset] = ((intValue8 >>> 14) | (intValue9 << 18)) & mask; + output[10 + outputOffset] = ((intValue9 >>> 12) | (intValue10 << 20)) & mask; + output[11 + outputOffset] = ((intValue10 >>> 10) | (intValue11 << 22)) & mask; + output[12 + outputOffset] = ((intValue11 >>> 8) | (intValue12 << 24)) & mask; + output[13 + outputOffset] = ((intValue12 >>> 6) | (intValue13 << 26)) & mask; + output[14 + outputOffset] = ((intValue13 >>> 4) | (intValue14 << 28)) & mask; + output[15 + outputOffset] = intValue14 >>> 2; + output[16 + outputOffset] = intValue15 & mask; + output[17 + outputOffset] = ((intValue15 >>> 30) | (intValue16 << 2)) & mask; + output[18 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[19 + outputOffset] = ((intValue17 >>> 26) | (intValue18 << 6)) & mask; + output[20 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[21 + outputOffset] = ((intValue19 >>> 22) | (intValue20 << 10)) & mask; + output[22 + outputOffset] = ((intValue20 >>> 20) | (intValue21 << 12)) & mask; + output[23 + outputOffset] = ((intValue21 >>> 18) | (intValue22 << 14)) & mask; + output[24 + outputOffset] = ((intValue22 >>> 16) | (intValue23 << 16)) & mask; + output[25 + outputOffset] = ((intValue23 >>> 14) | (intValue24 << 18)) & mask; + output[26 + outputOffset] = ((intValue24 >>> 12) | (intValue25 << 20)) & mask; + output[27 + outputOffset] = ((intValue25 >>> 10) | (intValue26 << 22)) & mask; + output[28 + outputOffset] = ((intValue26 >>> 8) | (intValue27 << 24)) & mask; + output[29 + outputOffset] = ((intValue27 >>> 6) | (intValue28 << 26)) & mask; + output[30 + outputOffset] = ((intValue28 >>> 4) | (intValue29 << 28)) & mask; + output[31 + outputOffset] = intValue29 >>> 2; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For31Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For31Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For31Decompress.java (revision 0) @@ -0,0 +1,103 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For31Decompress extends ForDecompress { + static final int numFrameBits = 31; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + int intValue9 = compressedBuffer.get(bufIndex + 9); + int intValue10 = compressedBuffer.get(bufIndex + 10); + int intValue11 = compressedBuffer.get(bufIndex + 11); + int intValue12 = compressedBuffer.get(bufIndex + 12); + int intValue13 = compressedBuffer.get(bufIndex + 13); + int intValue14 = compressedBuffer.get(bufIndex + 14); + int intValue15 = compressedBuffer.get(bufIndex + 15); + int intValue16 = compressedBuffer.get(bufIndex + 16); + int intValue17 = compressedBuffer.get(bufIndex + 17); + int intValue18 = compressedBuffer.get(bufIndex + 18); + int intValue19 = compressedBuffer.get(bufIndex + 19); + int intValue20 = compressedBuffer.get(bufIndex + 20); + int intValue21 = compressedBuffer.get(bufIndex + 21); + int intValue22 = compressedBuffer.get(bufIndex + 22); + int intValue23 = compressedBuffer.get(bufIndex + 23); + int intValue24 = compressedBuffer.get(bufIndex + 24); + int intValue25 = compressedBuffer.get(bufIndex + 25); + int intValue26 = compressedBuffer.get(bufIndex + 26); + int intValue27 = compressedBuffer.get(bufIndex + 27); + int intValue28 = compressedBuffer.get(bufIndex + 28); + int intValue29 = compressedBuffer.get(bufIndex + 29); + int intValue30 = compressedBuffer.get(bufIndex + 30); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = ((intValue0 >>> 31) | (intValue1 << 1)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 30) | (intValue2 << 2)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 29) | (intValue3 << 3)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 28) | (intValue4 << 4)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 27) | (intValue5 << 5)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 26) | (intValue6 << 6)) & mask; + output[7 + outputOffset] = ((intValue6 >>> 25) | (intValue7 << 7)) & mask; + output[8 + outputOffset] = ((intValue7 >>> 24) | (intValue8 << 8)) & mask; + output[9 + outputOffset] = ((intValue8 >>> 23) | (intValue9 << 9)) & mask; + output[10 + outputOffset] = ((intValue9 >>> 22) | (intValue10 << 10)) & mask; + output[11 + outputOffset] = ((intValue10 >>> 21) | (intValue11 << 11)) & mask; + output[12 + outputOffset] = ((intValue11 >>> 20) | (intValue12 << 12)) & mask; + output[13 + outputOffset] = ((intValue12 >>> 19) | (intValue13 << 13)) & mask; + output[14 + outputOffset] = ((intValue13 >>> 18) | (intValue14 << 14)) & mask; + output[15 + outputOffset] = ((intValue14 >>> 17) | (intValue15 << 15)) & mask; + output[16 + outputOffset] = ((intValue15 >>> 16) | (intValue16 << 16)) & mask; + output[17 + outputOffset] = ((intValue16 >>> 15) | (intValue17 << 17)) & mask; + output[18 + outputOffset] = ((intValue17 >>> 14) | (intValue18 << 18)) & mask; + output[19 + outputOffset] = ((intValue18 >>> 13) | (intValue19 << 19)) & mask; + output[20 + outputOffset] = ((intValue19 >>> 12) | (intValue20 << 20)) & mask; + output[21 + outputOffset] = ((intValue20 >>> 11) | (intValue21 << 21)) & mask; + output[22 + outputOffset] = ((intValue21 >>> 10) | (intValue22 << 22)) & mask; + output[23 + outputOffset] = ((intValue22 >>> 9) | (intValue23 << 23)) & mask; + output[24 + outputOffset] = ((intValue23 >>> 8) | (intValue24 << 24)) & mask; + output[25 + outputOffset] = ((intValue24 >>> 7) | (intValue25 << 25)) & mask; + output[26 + outputOffset] = ((intValue25 >>> 6) | (intValue26 << 26)) & mask; + output[27 + outputOffset] = ((intValue26 >>> 5) | (intValue27 << 27)) & mask; + output[28 + outputOffset] = ((intValue27 >>> 4) | (intValue28 << 28)) & mask; + output[29 + outputOffset] = ((intValue28 >>> 3) | (intValue29 << 29)) & mask; + output[30 + outputOffset] = ((intValue29 >>> 2) | (intValue30 << 30)) & mask; + output[31 + outputOffset] = intValue30 >>> 1; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For32Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For32Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For32Decompress.java (revision 0) @@ -0,0 +1,27 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +class For32Decompress extends ForDecompress { + static void decompressFrame(FrameOfRef frameOfRef) { + int oldBufPos = frameOfRef.compressedBuffer.position(); + frameOfRef.compressedBuffer.position(frameOfRef.COMPRESSED_INDEX); + frameOfRef.compressedBuffer.get(frameOfRef.unCompressedData, frameOfRef.offset, frameOfRef.unComprSize); + frameOfRef.compressedBuffer.position(oldBufPos); + } +} Index: src/java/org/apache/lucene/util/pfor/For3Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For3Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For3Decompress.java (revision 0) @@ -0,0 +1,75 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For3Decompress extends ForDecompress { + static final int numFrameBits = 3; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 3) & mask; + output[2 + outputOffset] = (intValue0 >>> 6) & mask; + output[3 + outputOffset] = (intValue0 >>> 9) & mask; + output[4 + outputOffset] = (intValue0 >>> 12) & mask; + output[5 + outputOffset] = (intValue0 >>> 15) & mask; + output[6 + outputOffset] = (intValue0 >>> 18) & mask; + output[7 + outputOffset] = (intValue0 >>> 21) & mask; + output[8 + outputOffset] = (intValue0 >>> 24) & mask; + output[9 + outputOffset] = (intValue0 >>> 27) & mask; + output[10 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[11 + outputOffset] = (intValue1 >>> 1) & mask; + output[12 + outputOffset] = (intValue1 >>> 4) & mask; + output[13 + outputOffset] = (intValue1 >>> 7) & mask; + output[14 + outputOffset] = (intValue1 >>> 10) & mask; + output[15 + outputOffset] = (intValue1 >>> 13) & mask; + output[16 + outputOffset] = (intValue1 >>> 16) & mask; + output[17 + outputOffset] = (intValue1 >>> 19) & mask; + output[18 + outputOffset] = (intValue1 >>> 22) & mask; + output[19 + outputOffset] = (intValue1 >>> 25) & mask; + output[20 + outputOffset] = (intValue1 >>> 28) & mask; + output[21 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[22 + outputOffset] = (intValue2 >>> 2) & mask; + output[23 + outputOffset] = (intValue2 >>> 5) & mask; + output[24 + outputOffset] = (intValue2 >>> 8) & mask; + output[25 + outputOffset] = (intValue2 >>> 11) & mask; + output[26 + outputOffset] = (intValue2 >>> 14) & mask; + output[27 + outputOffset] = (intValue2 >>> 17) & mask; + output[28 + outputOffset] = (intValue2 >>> 20) & mask; + output[29 + outputOffset] = (intValue2 >>> 23) & mask; + output[30 + outputOffset] = (intValue2 >>> 26) & mask; + output[31 + outputOffset] = intValue2 >>> 29; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For4Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For4Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For4Decompress.java (revision 0) @@ -0,0 +1,76 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For4Decompress extends ForDecompress { + static final int numFrameBits = 4; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 4) & mask; + output[2 + outputOffset] = (intValue0 >>> 8) & mask; + output[3 + outputOffset] = (intValue0 >>> 12) & mask; + output[4 + outputOffset] = (intValue0 >>> 16) & mask; + output[5 + outputOffset] = (intValue0 >>> 20) & mask; + output[6 + outputOffset] = (intValue0 >>> 24) & mask; + output[7 + outputOffset] = intValue0 >>> 28; + output[8 + outputOffset] = intValue1 & mask; + output[9 + outputOffset] = (intValue1 >>> 4) & mask; + output[10 + outputOffset] = (intValue1 >>> 8) & mask; + output[11 + outputOffset] = (intValue1 >>> 12) & mask; + output[12 + outputOffset] = (intValue1 >>> 16) & mask; + output[13 + outputOffset] = (intValue1 >>> 20) & mask; + output[14 + outputOffset] = (intValue1 >>> 24) & mask; + output[15 + outputOffset] = intValue1 >>> 28; + output[16 + outputOffset] = intValue2 & mask; + output[17 + outputOffset] = (intValue2 >>> 4) & mask; + output[18 + outputOffset] = (intValue2 >>> 8) & mask; + output[19 + outputOffset] = (intValue2 >>> 12) & mask; + output[20 + outputOffset] = (intValue2 >>> 16) & mask; + output[21 + outputOffset] = (intValue2 >>> 20) & mask; + output[22 + outputOffset] = (intValue2 >>> 24) & mask; + output[23 + outputOffset] = intValue2 >>> 28; + output[24 + outputOffset] = intValue3 & mask; + output[25 + outputOffset] = (intValue3 >>> 4) & mask; + output[26 + outputOffset] = (intValue3 >>> 8) & mask; + output[27 + outputOffset] = (intValue3 >>> 12) & mask; + output[28 + outputOffset] = (intValue3 >>> 16) & mask; + output[29 + outputOffset] = (intValue3 >>> 20) & mask; + output[30 + outputOffset] = (intValue3 >>> 24) & mask; + output[31 + outputOffset] = intValue3 >>> 28; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For5Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For5Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For5Decompress.java (revision 0) @@ -0,0 +1,77 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For5Decompress extends ForDecompress { + static final int numFrameBits = 5; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 5) & mask; + output[2 + outputOffset] = (intValue0 >>> 10) & mask; + output[3 + outputOffset] = (intValue0 >>> 15) & mask; + output[4 + outputOffset] = (intValue0 >>> 20) & mask; + output[5 + outputOffset] = (intValue0 >>> 25) & mask; + output[6 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[7 + outputOffset] = (intValue1 >>> 3) & mask; + output[8 + outputOffset] = (intValue1 >>> 8) & mask; + output[9 + outputOffset] = (intValue1 >>> 13) & mask; + output[10 + outputOffset] = (intValue1 >>> 18) & mask; + output[11 + outputOffset] = (intValue1 >>> 23) & mask; + output[12 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[13 + outputOffset] = (intValue2 >>> 1) & mask; + output[14 + outputOffset] = (intValue2 >>> 6) & mask; + output[15 + outputOffset] = (intValue2 >>> 11) & mask; + output[16 + outputOffset] = (intValue2 >>> 16) & mask; + output[17 + outputOffset] = (intValue2 >>> 21) & mask; + output[18 + outputOffset] = (intValue2 >>> 26) & mask; + output[19 + outputOffset] = ((intValue2 >>> 31) | (intValue3 << 1)) & mask; + output[20 + outputOffset] = (intValue3 >>> 4) & mask; + output[21 + outputOffset] = (intValue3 >>> 9) & mask; + output[22 + outputOffset] = (intValue3 >>> 14) & mask; + output[23 + outputOffset] = (intValue3 >>> 19) & mask; + output[24 + outputOffset] = (intValue3 >>> 24) & mask; + output[25 + outputOffset] = ((intValue3 >>> 29) | (intValue4 << 3)) & mask; + output[26 + outputOffset] = (intValue4 >>> 2) & mask; + output[27 + outputOffset] = (intValue4 >>> 7) & mask; + output[28 + outputOffset] = (intValue4 >>> 12) & mask; + output[29 + outputOffset] = (intValue4 >>> 17) & mask; + output[30 + outputOffset] = (intValue4 >>> 22) & mask; + output[31 + outputOffset] = intValue4 >>> 27; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For6Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For6Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For6Decompress.java (revision 0) @@ -0,0 +1,78 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For6Decompress extends ForDecompress { + static final int numFrameBits = 6; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 6) & mask; + output[2 + outputOffset] = (intValue0 >>> 12) & mask; + output[3 + outputOffset] = (intValue0 >>> 18) & mask; + output[4 + outputOffset] = (intValue0 >>> 24) & mask; + output[5 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[6 + outputOffset] = (intValue1 >>> 4) & mask; + output[7 + outputOffset] = (intValue1 >>> 10) & mask; + output[8 + outputOffset] = (intValue1 >>> 16) & mask; + output[9 + outputOffset] = (intValue1 >>> 22) & mask; + output[10 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[11 + outputOffset] = (intValue2 >>> 2) & mask; + output[12 + outputOffset] = (intValue2 >>> 8) & mask; + output[13 + outputOffset] = (intValue2 >>> 14) & mask; + output[14 + outputOffset] = (intValue2 >>> 20) & mask; + output[15 + outputOffset] = intValue2 >>> 26; + output[16 + outputOffset] = intValue3 & mask; + output[17 + outputOffset] = (intValue3 >>> 6) & mask; + output[18 + outputOffset] = (intValue3 >>> 12) & mask; + output[19 + outputOffset] = (intValue3 >>> 18) & mask; + output[20 + outputOffset] = (intValue3 >>> 24) & mask; + output[21 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[22 + outputOffset] = (intValue4 >>> 4) & mask; + output[23 + outputOffset] = (intValue4 >>> 10) & mask; + output[24 + outputOffset] = (intValue4 >>> 16) & mask; + output[25 + outputOffset] = (intValue4 >>> 22) & mask; + output[26 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[27 + outputOffset] = (intValue5 >>> 2) & mask; + output[28 + outputOffset] = (intValue5 >>> 8) & mask; + output[29 + outputOffset] = (intValue5 >>> 14) & mask; + output[30 + outputOffset] = (intValue5 >>> 20) & mask; + output[31 + outputOffset] = intValue5 >>> 26; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For7Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For7Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For7Decompress.java (revision 0) @@ -0,0 +1,79 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For7Decompress extends ForDecompress { + static final int numFrameBits = 7; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 7) & mask; + output[2 + outputOffset] = (intValue0 >>> 14) & mask; + output[3 + outputOffset] = (intValue0 >>> 21) & mask; + output[4 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask; + output[5 + outputOffset] = (intValue1 >>> 3) & mask; + output[6 + outputOffset] = (intValue1 >>> 10) & mask; + output[7 + outputOffset] = (intValue1 >>> 17) & mask; + output[8 + outputOffset] = (intValue1 >>> 24) & mask; + output[9 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[10 + outputOffset] = (intValue2 >>> 6) & mask; + output[11 + outputOffset] = (intValue2 >>> 13) & mask; + output[12 + outputOffset] = (intValue2 >>> 20) & mask; + output[13 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask; + output[14 + outputOffset] = (intValue3 >>> 2) & mask; + output[15 + outputOffset] = (intValue3 >>> 9) & mask; + output[16 + outputOffset] = (intValue3 >>> 16) & mask; + output[17 + outputOffset] = (intValue3 >>> 23) & mask; + output[18 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[19 + outputOffset] = (intValue4 >>> 5) & mask; + output[20 + outputOffset] = (intValue4 >>> 12) & mask; + output[21 + outputOffset] = (intValue4 >>> 19) & mask; + output[22 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[23 + outputOffset] = (intValue5 >>> 1) & mask; + output[24 + outputOffset] = (intValue5 >>> 8) & mask; + output[25 + outputOffset] = (intValue5 >>> 15) & mask; + output[26 + outputOffset] = (intValue5 >>> 22) & mask; + output[27 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[28 + outputOffset] = (intValue6 >>> 4) & mask; + output[29 + outputOffset] = (intValue6 >>> 11) & mask; + output[30 + outputOffset] = (intValue6 >>> 18) & mask; + output[31 + outputOffset] = intValue6 >>> 25; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For8Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For8Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For8Decompress.java (revision 0) @@ -0,0 +1,80 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For8Decompress extends ForDecompress { + static final int numFrameBits = 8; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 8) & mask; + output[2 + outputOffset] = (intValue0 >>> 16) & mask; + output[3 + outputOffset] = intValue0 >>> 24; + output[4 + outputOffset] = intValue1 & mask; + output[5 + outputOffset] = (intValue1 >>> 8) & mask; + output[6 + outputOffset] = (intValue1 >>> 16) & mask; + output[7 + outputOffset] = intValue1 >>> 24; + output[8 + outputOffset] = intValue2 & mask; + output[9 + outputOffset] = (intValue2 >>> 8) & mask; + output[10 + outputOffset] = (intValue2 >>> 16) & mask; + output[11 + outputOffset] = intValue2 >>> 24; + output[12 + outputOffset] = intValue3 & mask; + output[13 + outputOffset] = (intValue3 >>> 8) & mask; + output[14 + outputOffset] = (intValue3 >>> 16) & mask; + output[15 + outputOffset] = intValue3 >>> 24; + output[16 + outputOffset] = intValue4 & mask; + output[17 + outputOffset] = (intValue4 >>> 8) & mask; + output[18 + outputOffset] = (intValue4 >>> 16) & mask; + output[19 + outputOffset] = intValue4 >>> 24; + output[20 + outputOffset] = intValue5 & mask; + output[21 + outputOffset] = (intValue5 >>> 8) & mask; + output[22 + outputOffset] = (intValue5 >>> 16) & mask; + output[23 + outputOffset] = intValue5 >>> 24; + output[24 + outputOffset] = intValue6 & mask; + output[25 + outputOffset] = (intValue6 >>> 8) & mask; + output[26 + outputOffset] = (intValue6 >>> 16) & mask; + output[27 + outputOffset] = intValue6 >>> 24; + output[28 + outputOffset] = intValue7 & mask; + output[29 + outputOffset] = (intValue7 >>> 8) & mask; + output[30 + outputOffset] = (intValue7 >>> 16) & mask; + output[31 + outputOffset] = intValue7 >>> 24; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/For9Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For9Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For9Decompress.java (revision 0) @@ -0,0 +1,81 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For9Decompress extends ForDecompress { + static final int numFrameBits = 9; + static final int mask = (int) ((1L<= 32) { + int intValue0 = compressedBuffer.get(bufIndex); + int intValue1 = compressedBuffer.get(bufIndex + 1); + int intValue2 = compressedBuffer.get(bufIndex + 2); + int intValue3 = compressedBuffer.get(bufIndex + 3); + int intValue4 = compressedBuffer.get(bufIndex + 4); + int intValue5 = compressedBuffer.get(bufIndex + 5); + int intValue6 = compressedBuffer.get(bufIndex + 6); + int intValue7 = compressedBuffer.get(bufIndex + 7); + int intValue8 = compressedBuffer.get(bufIndex + 8); + output[0 + outputOffset] = intValue0 & mask; + output[1 + outputOffset] = (intValue0 >>> 9) & mask; + output[2 + outputOffset] = (intValue0 >>> 18) & mask; + output[3 + outputOffset] = ((intValue0 >>> 27) | (intValue1 << 5)) & mask; + output[4 + outputOffset] = (intValue1 >>> 4) & mask; + output[5 + outputOffset] = (intValue1 >>> 13) & mask; + output[6 + outputOffset] = (intValue1 >>> 22) & mask; + output[7 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[8 + outputOffset] = (intValue2 >>> 8) & mask; + output[9 + outputOffset] = (intValue2 >>> 17) & mask; + output[10 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[11 + outputOffset] = (intValue3 >>> 3) & mask; + output[12 + outputOffset] = (intValue3 >>> 12) & mask; + output[13 + outputOffset] = (intValue3 >>> 21) & mask; + output[14 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[15 + outputOffset] = (intValue4 >>> 7) & mask; + output[16 + outputOffset] = (intValue4 >>> 16) & mask; + output[17 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask; + output[18 + outputOffset] = (intValue5 >>> 2) & mask; + output[19 + outputOffset] = (intValue5 >>> 11) & mask; + output[20 + outputOffset] = (intValue5 >>> 20) & mask; + output[21 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[22 + outputOffset] = (intValue6 >>> 6) & mask; + output[23 + outputOffset] = (intValue6 >>> 15) & mask; + output[24 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[25 + outputOffset] = (intValue7 >>> 1) & mask; + output[26 + outputOffset] = (intValue7 >>> 10) & mask; + output[27 + outputOffset] = (intValue7 >>> 19) & mask; + output[28 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[29 + outputOffset] = (intValue8 >>> 5) & mask; + output[30 + outputOffset] = (intValue8 >>> 14) & mask; + output[31 + outputOffset] = intValue8 >>> 23; + inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} Index: src/java/org/apache/lucene/util/pfor/ForDecompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0) @@ -0,0 +1,56 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; + +/** PFor frame decompression for any number of frame bits. */ +class ForDecompress { + + static void decodeAnyFrame( + IntBuffer intBuffer, int bufIndex, int inputSize, int numFrameBits, + int[] output, int outputOffset) { + + assert numFrameBits > 0 : numFrameBits; + assert numFrameBits <= 32 : numFrameBits; + final int mask = (int) ((1L<>> bitPos) & mask; + if (--inputSize == 0) return; + bitPos += numFrameBits; + } + + int intValue2 = intBuffer.get(++bufIndex); + output[++outputOffset] = ( (bitPos == 32) + ? intValue2 + : ((intValue1 >>> bitPos) | (intValue2 << (32 - bitPos))) + ) & mask; + + if (--inputSize == 0) return; + + intValue1 = intValue2; + bitPos += numFrameBits - 32; + } while (true); + } +} Index: src/java/org/apache/lucene/util/pfor/FrameOfRef.java =================================================================== --- src/java/org/apache/lucene/util/pfor/FrameOfRef.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/FrameOfRef.java (revision 0) @@ -0,0 +1,323 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +import java.util.Arrays; + +import org.apache.lucene.util.BitUtil; + +/** Frame of Reference lossless integer compression/decompression. + * For positive integers, the compression is done by leaving out + * the most significant bits, and storing all numbers with a fixed number of bits + * contiguously in a buffer of bits. This buffer is called the frame, and it + * can store positive numbers in a range from 0 to a constant maximum fitting in + * the number of bits available for a single compressed number. + *

+ * This implementation uses 0 as the lower bound reference for the frame, + * so small positive integers can be most effectively compressed. + *

+ * Optimized code is used for decompression, see class ForDecompress and its subclasses. + *
Use of the -server option helps performance for the Sun 1.6 jvm under Linux. + *

+ * This class does not provide delta coding because the Lucene index + * structures already have that. + *

+ * To be done: + *

    + *
  • + * Optimize compression code by specializing for number of frame bits. + *
  • + * IntBuffer.get() is somewhat faster that IntBuffer.get(index), adapt (de)compression to + * use the relative get() method. + *
  • + * Check javadoc generation and generated javadocs. Add javadoc code references. + *
+ */ +public class FrameOfRef { + /** Number of frame bits. 2**numFrameBits - 1 is the maximum compressed value. */ + protected int numFrameBits; + + /** Constant header tag to allow other compression methods, use value 0001 for + * Frame of reference. + * CHECKME: Move this to another class defining various decompression methods? + */ + protected int compressionMethod; + private final int FOR_COMPRESSION = 1; /** encode compression method in header */ + + /** IntBuffer for compressed data */ + protected IntBuffer compressedBuffer; + + /** Index of header in int buffer */ + protected final int HEADER_INDEX = 0; + + /** Start index in int buffer of array integers each compressed to numFrameBits. */ + protected final int COMPRESSED_INDEX = HEADER_INDEX + 1; + protected final int HEADER_SIZE = 1; // one integer in IntBuffer + + + /** Uncompressed data */ + protected int[] unCompressedData; + /** Offset into unCompressedData */ + protected int offset; + /** Size of unCompressedData, -1 when not available. */ + protected int unComprSize = -1; + + /** Create a Frame of Reference integer compressor/decompressor. */ + public FrameOfRef() { + } + + /** Integer buffer to hold the compressed data.
+ * Compression and decompression do not affect the current buffer position, + * and the beginning of the compressed data should be or will be at the current + * buffer position.
+ * When the buffer is not large enough, ArrayIndexOutOfBoundExceptions will occur + * during compression/decompression.
+ * Without a buffer for compressed data, compress() will only determine the number + * of integers needed in the buffer, see compress().
+ * Without a valid buffer, decompress() will throw a NullPointerException.
+ * For optimal speed when the IntBuffer is a view on a ByteBuffer, + * the IntBuffer should have a byte offset of a multiple of 4 bytes, possibly 0.
+ * An IntBuffer is used here because 32 bits can efficiently accessed in the buffer + * on all current processors, and a positive int is normally large enough + * for various purposes in a Lucene index. + * + * @param compressedBuffer The buffer to hold the compressed integers. + * + */ + public void setCompressedBuffer(IntBuffer compressedBuffer) { + this.compressedBuffer = compressedBuffer; + } + + + /** Array with offset holding uncompressed data. + * @param unCompressedData The array holding uncompressed integers. + * @param offset offset in unCompressedData. + * @param unComprSize The number of uncompressed integers, should be at least 1. + */ + public void setUnCompressedData(int[] unCompressedData, int offset, int unComprSize) { + assert unCompressedData != null; + assert offset >= 0; + assert unComprSize >= 1; + assert (offset + unComprSize) <= unCompressedData.length; + this.unCompressedData = unCompressedData; + this.offset = offset; + this.unComprSize = unComprSize; + } + + /** Compress the uncompressed data into the buffer using the given number of + * frame bits, storing only this number of least significant bits of the + * uncompressed integers in the compressed buffer. + * Should only be used after setUnCompressedData(). + *
+ * When setCompressBuffer() was not done, no actual compression is done. + * Regardless of the use of setCompressBuffer(), bufferByteSize() will return + * a valid value after calling compress(). + *

+ * When a buffer is available, the following is done. + * A header is stored as a first integer into the buffer, encoding + * the compression method, the number of frame bits and the number of compressed integers. + * All uncompressed integers are stored sequentially in compressed form + * in the buffer after the header. + * + * @param numFrameBits The number of frame bits. Should be between 1 and 32. + * Note that when this value is 32, no compression occurs. + */ + public void compress(int numFrameBits) { + assert numFrameBits >= 1; + assert numFrameBits <= 32; + this.numFrameBits = numFrameBits; + encodeHeader(unComprSize); + for (int i = 0; i < unComprSize; i++) { + int v = unCompressedData[i + offset]; + encodeCompressedValue(i, v); + } + } + + /** As compress(), using the result of frameBitsForCompression() as the number of frame bits. */ + public void compress() { + compress( frameBitsForCompression()); + } + + /** Return the number of integers used in IntBuffer. + * Only valid after compress() or decompress(). + */ + public int compressedSize() { + return HEADER_SIZE + (unComprSize * numFrameBits + 31) / 32; + } + + /** Encode an integer value by compressing it into the buffer. + * @param compressedPos The index of the compressed integer in the compressed buffer. + * @param value The non negative value to be stored in compressed form. + * This should fit into the number of frame bits. + */ + protected void encodeCompressedValue(int compressedPos, int value) { + encodeCompressedValueBase(compressedPos, value, numFrameBits); // FIXME: inline private method. + } + + /** Encode a value into the compressed buffer. + * Since numFrameBits is always smaller than the number of bits in an int, + * at most two ints in the buffer will be affected. + *
Has no effect when compressedBuffer == null. + *
This could be specialized for numBits just like decompressFrame(). + */ + private void encodeCompressedValueBase(int compressedPos, int value, int numBits) { + assert numBits >= 1; + assert numBits <= 32; + if (compressedBuffer == null) { + return; + } + final int mask = (int) ((1L << numBits) - 1); + assert ((value & mask) == value) : ("value " + value + ", mask " + mask + ", numBits " + numBits); // lossless compression + final int compressedBitPos = numBits * compressedPos; + final int firstBitPosition = compressedBitPos & 31; + int intIndex = COMPRESSED_INDEX + (compressedBitPos >> 5); + setBufferIntBits(intIndex, firstBitPosition, numBits, value); + if ((firstBitPosition + numBits) > 32) { // value does not fit in first int + setBufferIntBits(intIndex+1, 0, (firstBitPosition + numBits - 32), (value >>> (32 - firstBitPosition))); + } + } + + /** Change bits of an integer in the compressed buffer. + *
A more efficient implementation is possible when the compressed + * buffer is known to contain only zero bits, in that case one mask operation can be removed. + * @param intIndex The index of the affected integer in the compressed buffer. + * @param firstBitPosition The position of the least significant bit to be changed. + * @param numBits The number of more significant bits to be changed. + * @param value The new value of the bits to be changed, with the least significant bit at position zero. + */ + protected void setBufferIntBits(int intIndex, int firstBitPosition, int numBits, int value) { + final int mask = (int) ((1L << numBits) - 1); + compressedBuffer.put(intIndex, + (compressedBuffer.get(intIndex) + & ~ (mask << firstBitPosition)) // masking superfluous on clear buffer + | (value << firstBitPosition)); + } + + /** The 4 byte header (32 bits) contains: + *

    + *
  • + *
      + *
    • 4 bits for the compression method: 0b0001 for FrameOfRef, + *
    • 4 bits unused, + *
    + *
  • + *
      + *
    • 5 bits for (numFrameBits-1), + *
    • 3 bit unused, + *
    + *
  • 8 bits for number of compressed integers - 1, + *
  • 8 bits unused. + *
+ */ + private void encodeHeader(int unComprSize) { + assert numFrameBits >= 1; + assert numFrameBits <= (1 << 5); // 32 + assert unComprSize >= 1; + assert unComprSize <= (1 << 8); // 256 + if (compressedBuffer != null) { + compressedBuffer.put(HEADER_INDEX, + ((unComprSize-1) << 16) + | ((numFrameBits-1) << 8) + | (FOR_COMPRESSION << 4)); + } + } + + protected void decodeHeader() { + int header = compressedBuffer.get(HEADER_INDEX); + unComprSize = ((header >>> 16) & 255) + 1; + numFrameBits = ((header >>> 8) & 31) + 1; + compressionMethod = (header >>> 4) & 15; + assert compressionMethod == FOR_COMPRESSION : compressionMethod; + } + + /** Decompress from the buffer into output from a given offset. */ + public void decompress() { + decodeHeader(); + decompressFrame(); + } + + /** Return the number of integers available for decompression. + * Do not use before an IntBuffer was passed to setCompressBuffer. + */ + public int decompressedSize() { + decodeHeader(); + return unComprSize; + } + + /** For performance, this delegates to classes with fixed numFrameBits. */ + private void decompressFrame() { + switch (numFrameBits) { + // CHECKME: two other implementations might be faster: + // - array of static methods: Method[numFrameBits].invoke(null, [this]), + // - array of non static decompressors: ForDecompressor[numFrameBits].decompressFrame(this) . + case 1: For1Decompress.decompressFrame(this); break; + case 2: For2Decompress.decompressFrame(this); break; + case 3: For3Decompress.decompressFrame(this); break; + case 4: For4Decompress.decompressFrame(this); break; + case 5: For5Decompress.decompressFrame(this); break; + case 6: For6Decompress.decompressFrame(this); break; + case 7: For7Decompress.decompressFrame(this); break; + case 8: For8Decompress.decompressFrame(this); break; + case 9: For9Decompress.decompressFrame(this); break; + case 10: For10Decompress.decompressFrame(this); break; + case 11: For11Decompress.decompressFrame(this); break; + case 12: For12Decompress.decompressFrame(this); break; + case 13: For13Decompress.decompressFrame(this); break; + case 14: For14Decompress.decompressFrame(this); break; + case 15: For15Decompress.decompressFrame(this); break; + case 16: For16Decompress.decompressFrame(this); break; + case 17: For17Decompress.decompressFrame(this); break; + case 18: For18Decompress.decompressFrame(this); break; + case 19: For19Decompress.decompressFrame(this); break; + case 20: For20Decompress.decompressFrame(this); break; + case 21: For21Decompress.decompressFrame(this); break; + case 22: For22Decompress.decompressFrame(this); break; + case 23: For23Decompress.decompressFrame(this); break; + case 24: For24Decompress.decompressFrame(this); break; + case 25: For25Decompress.decompressFrame(this); break; + case 26: For26Decompress.decompressFrame(this); break; + case 27: For27Decompress.decompressFrame(this); break; + case 28: For28Decompress.decompressFrame(this); break; + case 29: For29Decompress.decompressFrame(this); break; + case 30: For30Decompress.decompressFrame(this); break; + case 31: For31Decompress.decompressFrame(this); break; + case 32: For32Decompress.decompressFrame(this); break; + default: + throw new IllegalStateException("Unknown number of frame bits " + numFrameBits); + } + } + + public int getNumFrameBits() { + return numFrameBits; + } + + /** Determine the number of frame bits to be used for compression. + * Use only after setUnCompressedData(). + * @return The number of bits needed to encode the maximum positive uncompressed value. + * Negative uncompressed values have no influence on the result. + */ + public int frameBitsForCompression() { + int maxNonNegVal = 0; + for (int i = offset; i < (offset + unComprSize); i++) { + if (unCompressedData[i] > maxNonNegVal) { + maxNonNegVal = unCompressedData[i]; + } + } + return BitUtil.logNextHigherPowerOfTwo(maxNonNegVal) + 1; + } +} Index: src/java/org/apache/lucene/util/pfor/PFor.java =================================================================== --- src/java/org/apache/lucene/util/pfor/PFor.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/PFor.java (revision 0) @@ -0,0 +1,414 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Arrays; + +/** Patched Frame of Reference PFOR compression/decompression. + *

+ * As defined in:
+ * Super-Scalar RAM-CPU Cache Compression
+ * Marcin Zukowski, Sándor Héman, Niels Nes, Peter Boncz, 2006.
+ * with extensions from:
+ * Performance of Compressed Inverted List Caching in Search Engines
+ * Jiangong Zhang, Xiaohui Long, Torsten Suel, 2008.
+ *

+ * This class does not provide delta coding because the lucene index + * structures already have that. + *

+ * The implementation uses 0 as lower bound for the frame, + * so small positive integers will be most effectively compressed. + *

+ * Some optimized code is used for decompression, + * see class ForDecompress and its subclasses. + *
Good decompression performance will depend on the performance + * of java.nio.IntBuffer indexed get() methods. + *
Use of the -server option helps performance for the Sun 1.6 jvm under Linux. + *

+ * The start point of first exception is at its natural boundary: + * 2 byte exceptions at even byte position, 4 byte at quadruple. + *

+ * To be done: + *

    + *
  • + * Optimize compression code. + *
  • + * IntBuffer.get() is somewhat faster that IntBuffer.get(index), adapt (de)compression for to + * use the relative get() method. + *
  • + * Check javadoc generation and generated javadocs. Add javadoc code references. + *
+ */ +public class PFor extends FrameOfRef { + /** Index on input and in compressed frame of first exception, -1 when no exceptions */ + private int firstExceptionIndex; + + /** CHECKME: Move this to another class defining various decompression methods? */ + private final int PFOR_COMPRESSION = 2; /** to encode compression method in header */ + + /** How to encode PFor exceptions: 0: byte, 1: short, 2:int, unused: 3: long */ + private int exceptionCode = -1; + + /** Total number of exception values */ + private int numExceptions; + + /** Create a PFor compressor/decompressor. */ + public PFor() { + } + + /** Compress the decompressed data into the buffer. + * Should only be used after setUnCompressedData(). + *
+ * When setCompressBuffer() was not done, no actual compression is done. + * Regardless of the use of setCompressBuffer(), bufferByteSize() will return + * a valid value after calling compress(). + *

+ * When a buffer is available, the following is done. + * A header is stored into the buffer, encoding a.o. numFrameBits and unComprSize. + * All ints < 2**numFrameBits are stored sequentially in compressed form + * in the buffer. + * All other ints are stored in the buffer as exceptions after the compressed sequential ints, + * using 1, 2 or 4 bytes per exception, starting at the first byte after the compressed + * sequential ints. + *
+ * The index of the first exception is encoded in the header in the buffer, + * all later exceptions have the offset to the next exception as their value, + * the last one offset to just after the available input size. + * After the first exception, when the next exception index does not fit in + * numFrameBits bits, an exception after 2**numFrameBits inputs is forced and inserted. + *
+ * Exception values are stored in the order of the exceptions. + * The number of bytes used for an exception is also encoded in the header. + * This depends on the maximum exception value and does not vary between the exceptions. + */ + public void compress(int numFrameBits) { + assert numFrameBits >= 1; + assert numFrameBits <= 32; + this.numFrameBits = numFrameBits; + numExceptions = 0; + int maxException = -1; + firstExceptionIndex = -1; + int lastExceptionIndex = -1; + int i; + int[] exceptionValues = new int[unComprSize]; + int maxNonExceptionMask = (int) ((1L << numFrameBits) - 1); + int maxChain = 254; // maximum value of firstExceptionIndex in header + // CHECKME: maxChain 1 off because of initial value of lastExceptionIndex and force exception test below? + for (i = 0; i < unComprSize; i++) { + int v = unCompressedData[i + offset]; + // FIXME: split this loop to avoid if statement in loop. + // use predication for this: (someBool ? 1 : 0), and hope that the jit optimizes this. + if ( (((v & maxNonExceptionMask) == v) // no value exception + && (i < (lastExceptionIndex + maxChain)))) { // no forced exception + encodeCompressedValue(i, v); // normal encoding + } else { // exception + exceptionValues[numExceptions] = v; + numExceptions++; + if (firstExceptionIndex == -1) { + firstExceptionIndex = i; + assert firstExceptionIndex <= 254; // maximum value of firstExceptionIndex in header + maxException = v; + maxChain = 1 << ((30 < numFrameBits) ? 30 : numFrameBits); // fewer bits available for exception chain value. + } else if (v > maxException) { + maxException = v; + } + // encode the previous exception pointer + if (lastExceptionIndex >= 0) { + encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); + } + lastExceptionIndex = i; + } + } + if (lastExceptionIndex >= 0) { + encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); // end the exception chain. + } + int bitsInArray = numFrameBits * unCompressedData.length; + int bytesInArray = (bitsInArray + 7) / 8; + if (maxException < (1 << 8)) { // exceptions as byte + exceptionCode = 0; + } else if (maxException < (1 << 16)) { // exceptions as 2 bytes + exceptionCode = 1; + } else /* if (maxException < (1L << 32)) */ { // exceptions as 4 bytes + exceptionCode = 2; + } + encodeHeader(unComprSize, firstExceptionIndex); + encodeExceptionValues(exceptionValues); + } + + /** Return the number bytes used for a single exception */ + private int exceptionByteSize() { + assert exceptionCode >= 0; + assert exceptionCode <= 2; + return exceptionCode == 0 ? 1 + : exceptionCode == 1 ? 2 + : 4; + } + + /** Return the number of exceptions. + * Only valid after compress() or decompress(). + */ + public int getNumExceptions() { + return numExceptions; + } + + private int compressedArrayByteSize() { + int compressedArrayBits = unComprSize * numFrameBits; + return (compressedArrayBits + 7) / 8; + } + + /** Return the number of integers used in IntBuffer. + * Only valid after compress() or decompress(). + */ + public int compressedSize() { + // numExceptions only valid after compress() or decompress() + return HEADER_SIZE + + ((compressedArrayByteSize() + + exceptionByteSize() * numExceptions + + 3) >> 2); // round up to next multiple of 4 and divide by 4 + } + + private void encodeExceptionValues(int[] exceptionValues) { + if ((compressedBuffer == null) || (numExceptions == 0)) { + return; + } + int excByteOffset = compressedArrayByteSize(); + + switch (exceptionCode) { + case 0: { // 1 byte exceptions + int i = 0; + do { + int intIndex = COMPRESSED_INDEX + (excByteOffset >> 2); // round down here. + setBufferIntBits(intIndex, ((excByteOffset & 3) * 8), 8, exceptionValues[i]); + excByteOffset++; + } while (++i < numExceptions); + } + break; + + case 1: { // 2 byte exceptions + int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes. + int intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here. + int i = 0; + if ((excShortOffset & 1) != 0) { // encode first 2 byte exception in high 2 bytes of same int as last frame bits. + setBufferIntBits(intIndex, 16, 16, exceptionValues[i]); + intIndex++; + i++; + } + for (; i < (numExceptions-1); i += 2) { + compressedBuffer.put(intIndex++, (exceptionValues[i+1] << 16) | exceptionValues[i]); + } + if (i < numExceptions) { + compressedBuffer.put(intIndex, exceptionValues[i]); // also clear the high 16 bits + } + } + break; + + case 2: { // 4 byte exceptions + int excIntOffSet = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints. + int i = 0; + do { + compressedBuffer.put(excIntOffSet + i, exceptionValues[i]); + } while(++i < numExceptions); + } + break; + } + } + + /** Decode the exception values while going through the exception chain. + *
For performance, delegate/subclass this to classes with fixed exceptionCode. + *
Also, decoding exceptions is preferably done from an int border instead of + * from a random byte directly after the compressed array. This will allow faster + * decoding of exceptions, at the cost of at most 3 bytes. + *
When ((numFrameBits * unComprSize) % 32) == 0, this cost will always be + * zero bytes so specialize for these cases. + */ + private void patchExceptions() { + numExceptions = 0; + if (firstExceptionIndex == -1) { + return; + } + int excIndex = firstExceptionIndex; + int excByteOffset = compressedArrayByteSize(); + int excValue; + int intIndex; + + switch (exceptionCode) { + case 0: { // 1 byte exceptions + do { + intIndex = COMPRESSED_INDEX + (excByteOffset >> 2); + int firstBitPosition = (excByteOffset & 3) << 3; + excValue = (compressedBuffer.get(intIndex) >>> firstBitPosition) & ((1 << 8) - 1); + excIndex = patch(excIndex, excValue); + excByteOffset++; + } while (excIndex < unComprSize); + } + break; + + case 1: { // 2 byte exceptions + int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes. + intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here. + int i = 0; + if ((excShortOffset & 1) != 0) { + // decode first 2 byte exception from high 2 bytes of same int as last frame bits. + excValue = compressedBuffer.get(intIndex++) >>> 16; + excIndex = patch(excIndex, excValue); + } + while (excIndex < unComprSize) { + excValue = compressedBuffer.get(intIndex) & ((1<<16)-1); + excIndex = patch(excIndex, excValue); + if (excIndex >= unComprSize) { + break; + } + excValue = compressedBuffer.get(intIndex++) >>> 16; + excIndex = patch(excIndex, excValue); + } + } + break; + + case 2: // 4 byte exceptions + intIndex = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints. + do { + excValue = compressedBuffer.get(intIndex++); + excIndex = patch(excIndex, excValue); + } while (excIndex < unComprSize); + break; + } + } + + /** The 4 byte header (32 bits) contains: + * + * - 4 bits for the compression method: 0b0001 for PFor + * - 4 bits unused + * + * - 5 bits for (numFrameBits-1) + * - 2 bits for the exception code: 0b00: byte, 0b01: short, 0b10: int, 0b11: long (unused). + * - 1 bit unused + * + * - 8 bits for uncompressed input size - 1, + * + * - 8 bits for the index of the first exception + 1, (0 when no exceptions) + */ + private void encodeHeader(int unComprSize, int firstExceptionIndex) { + assert exceptionCode >= 0; + assert exceptionCode <= 2; // 3 for long, but unused for now. + assert numFrameBits >= 1; + assert numFrameBits <= 32; + assert unComprSize >= 1; + assert unComprSize <= 128; + assert firstExceptionIndex >= -1; + assert firstExceptionIndex < unComprSize; + if (compressedBuffer != null) { + compressedBuffer.put(HEADER_INDEX, + ((firstExceptionIndex+1) << 24) + | ((unComprSize-1) << 16) + | ((exceptionCode & 3) << 13) | ((numFrameBits-1) << 8) + | (PFOR_COMPRESSION << 4)); + } + } + + protected void decodeHeader() { + int header = compressedBuffer.get(HEADER_INDEX); + firstExceptionIndex = ((header >>> 24) & 255) - 1; + unComprSize = ((header >>> 16) & 255) + 1; + numFrameBits = ((header >>> 8) & 31) + 1; + assert numFrameBits > 0: numFrameBits; + assert numFrameBits <= 32: numFrameBits; + compressionMethod = (header >>> 4) & 15; + assert compressionMethod == PFOR_COMPRESSION : compressionMethod; + exceptionCode = (header >>> 13) & 3; + assert exceptionCode <= 2; + } + + /** Decompress from the buffer into output from a given offset. */ + public void decompress() { + super.decompress(); + patchExceptions(); + } + + /** Patch and return index of next exception */ + private int patch(int excIndex, int excValue) { + int nextExceptionIndex = unCompressedData[excIndex] + excIndex + 1; // chain offset + unCompressedData[excIndex + offset] = excValue; // patch + assert nextExceptionIndex > excIndex; + numExceptions++; + return nextExceptionIndex; + } + + /** Determine the number of frame bits to be used for compression. + * Use only after setUnCompressedData(). + * This is done by taking a copy of the input, sorting it and using this + * to determine the compressed size for each possible numbits in a single pass, + * ignoring forced exceptions. + * Finally an estimation of the number of forced exceptions is reduced to + * less than 1 in 32 input numbers by increasing the number of frame bits. + * This implementation works by determining the total number of bytes needed for + * the compressed data, but does take into account alignment of exceptions + * at 2 or 4 byte boundaries. + */ + public int frameBitsForCompression() { + if ((offset + unComprSize) > unCompressedData.length) { + throw new IllegalArgumentException( "(offset " + offset + + " + unComprSize " + unComprSize + + ") > unCompressedData.length " + unCompressedData.length); + } + int copy[] = Arrays.copyOfRange(unCompressedData, offset, offset + unComprSize); + assert copy.length == unComprSize; + Arrays.sort(copy); + int maxValue = copy[copy.length-1]; + if (maxValue <= 1) { + return 1; + } + int bytesPerException = (maxValue < (1 << 8)) ? 1 : (maxValue < (1 << 16)) ? 2 : 4; + int frameBits = 1; + int bytesForFrame = (copy.length * frameBits + 7) / 8; + // initially assume all input is an exception. + int totalBytes = bytesForFrame + copy.length * bytesPerException; // excluding the header. + int bestBytes = totalBytes; + int bestFrameBits = frameBits; + int bestExceptions = copy.length; + for (int i = 0; i < copy.length; i++) { + // determine frameBits so that copy[i] is no more exception + while (copy[i] >= (1 << frameBits)) { + if (frameBits == 30) { // no point to increase further. + return bestFrameBits; + } + ++frameBits; + // increase bytesForFrame and totalBytes to correspond to frameBits + int newBytesForFrame = (copy.length * frameBits + 7) / 8; + totalBytes += newBytesForFrame - bytesForFrame; + bytesForFrame = newBytesForFrame; + } + totalBytes -= bytesPerException; // no more need to store copy[i] as exception + if (totalBytes <= bestBytes) { // <= : prefer fewer exceptions at higher number of frame bits. + bestBytes = totalBytes; + bestFrameBits = frameBits; + bestExceptions = (copy.length - i - 1); + } + } + if (bestExceptions > 0) { // check for forced exceptions. + // This ignores the position of the first exception, for which enough bits are available in the header. + int allowedNumExceptions = bestExceptions + (copy.length >> 5); // 1 in 32 is allowed to be forced. + // (copy.length >> bestFrameBits): Minimum exception chain size including forced ones, + // ignoring the position of the first exception. + while (allowedNumExceptions < (copy.length >> bestFrameBits)) { // Too many forced? + bestFrameBits++; // Reduce forced exceptions and perhaps reduce actual exceptions + // Dilemma: decompression speed reduces with increasing number of frame bits, + // so it may be better to increase no more than once or twice here. + } + } + return bestFrameBits; + } +} Index: src/java/org/apache/lucene/util/pfor/gendecompress.py =================================================================== --- src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0) +++ src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0) @@ -0,0 +1,110 @@ +""" + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +""" +Generate source code for java classes for FOR decompression. +""" + +def bitsExpr(i, numFrameBits): + framePos = i * numFrameBits + intValNum = (framePos / 32) + bitPos = framePos % 32 + bitsInInt = "intValue" + str(intValNum) + needBrackets = 0 + if bitPos > 0: + bitsInInt += " >>> " + str(bitPos) + needBrackets = 1 + if bitPos + numFrameBits > 32: + if needBrackets: + bitsInInt = "(" + bitsInInt + ")" + bitsInInt += " | (intValue" + str(intValNum+1) + " << "+ str(32 - bitPos) + ")" + needBrackets = 1 + if bitPos + numFrameBits != 32: + if needBrackets: + bitsInInt = "(" + bitsInInt + ")" + bitsInInt += " & mask" + return bitsInInt + + +def genDecompressClass(numFrameBits): + className = "For" + str(numFrameBits) + "Decompress" + fileName = className + ".java" + imports = "import java.nio.IntBuffer;\n" + f = open(fileName, 'w') + w = f.write + try: + w("package org.apache.lucene.util.pfor;\n") + w("""/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */""") + w("\n/* This program is generated, do not modify. See gendecompress.py */\n\n") + w("import java.nio.IntBuffer;\n") + w("class " + className + " extends ForDecompress {\n") + w(" static final int numFrameBits = " + str(numFrameBits) + ";\n") + w(" static final int mask = (int) ((1L<= 32) {\n") + for i in range(numFrameBits): # declare int vars and init from buffer + w(" int intValue" + str(i) + " = compressedBuffer.get(bufIndex") + if i > 0: + w(" + " + str(i)) + w(");\n") + for i in range(32): # set output from int vars + w(" output[" + str(i) + " + outputOffset] = " + bitsExpr(i, numFrameBits) + ";\n") + w(""" inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} +""") + finally: f.close() + + + +def genDecompressClasses(): + numFrameBits = 1 + while numFrameBits <= 31: # 32 special case, not generated. + genDecompressClass(numFrameBits) + numFrameBits += 1 + + + +if __name__ == "__main__": + genDecompressClasses() Index: src/java/org/apache/lucene/util/pfor/package.html =================================================================== --- src/java/org/apache/lucene/util/pfor/package.html (revision 0) +++ src/java/org/apache/lucene/util/pfor/package.html (revision 0) @@ -0,0 +1,25 @@ + + + + + + + +Classes dealing with (patched) frame of reference compression and decompression. + + Index: src/test/org/apache/lucene/TestSearchForDuplicates.java =================================================================== --- src/test/org/apache/lucene/TestSearchForDuplicates.java (revision 822088) +++ src/test/org/apache/lucene/TestSearchForDuplicates.java (working copy) @@ -94,6 +94,9 @@ for (int j = 0; j < MAX_DOCS; j++) { Document d = new Document(); d.add(new Field(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES, Field.Index.ANALYZED)); + + // NOTE: this ID_FIELD produces no tokens since + // SimpleAnalyzer discards numbers d.add(new Field(ID_FIELD, Integer.toString(j), Field.Store.YES, Field.Index.ANALYZED)); writer.addDocument(d); } Index: src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java =================================================================== --- src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (revision 822088) +++ src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (working copy) @@ -27,6 +27,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.MockRAMDirectory; +import org.apache.lucene.util._TestUtil; import org.apache.lucene.search.PhraseQuery; @@ -45,6 +46,7 @@ addDocs(writer, 100); assertEquals(100, writer.docCount()); writer.close(); + _TestUtil.checkIndex(dir); writer = newWriter(aux, true); writer.setUseCompoundFile(false); // use one without a compound file @@ -65,6 +67,7 @@ writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); assertEquals(190, writer.docCount()); writer.close(); + _TestUtil.checkIndex(dir); // make sure the old index is correct verifyNumDocs(aux, 40); @@ -125,12 +128,13 @@ public void testWithPendingDeletes() throws IOException { // main directory - Directory dir = new RAMDirectory(); + Directory dir = new MockRAMDirectory(); // auxiliary directory - Directory aux = new RAMDirectory(); + Directory aux = new MockRAMDirectory(); setUpDirs(dir, aux); IndexWriter writer = newWriter(dir, false); + writer.addIndexesNoOptimize(new Directory[] {aux}); // Adds 10 docs, then replaces them with another 10 Index: src/test/org/apache/lucene/index/TestAtomicUpdate.java =================================================================== --- src/test/org/apache/lucene/index/TestAtomicUpdate.java (revision 822088) +++ src/test/org/apache/lucene/index/TestAtomicUpdate.java (working copy) @@ -126,8 +126,8 @@ TimedThread[] threads = new TimedThread[4]; IndexWriter writer = new MockIndexWriter(directory, true, ANALYZER, true); - writer.setMaxBufferedDocs(7); - writer.setMergeFactor(3); + writer.setMaxBufferedDocs(4); + writer.setMergeFactor(2); // Establish a base index of 100 docs: for(int i=0;i<100;i++) { @@ -142,33 +142,34 @@ assertEquals(100, r.numDocs()); r.close(); + int upto = 0; + IndexerThread indexerThread = new IndexerThread(writer, threads); - threads[0] = indexerThread; + threads[upto++] = indexerThread; indexerThread.start(); - IndexerThread indexerThread2 = new IndexerThread(writer, threads); - threads[1] = indexerThread2; - indexerThread2.start(); + //IndexerThread indexerThread2 = new IndexerThread(writer, threads); + //threads[upto++] = indexerThread2; + //indexerThread2.start(); SearcherThread searcherThread1 = new SearcherThread(directory, threads); - threads[2] = searcherThread1; + threads[upto++] = searcherThread1; searcherThread1.start(); - SearcherThread searcherThread2 = new SearcherThread(directory, threads); - threads[3] = searcherThread2; - searcherThread2.start(); + //SearcherThread searcherThread2 = new SearcherThread(directory, threads); + //threads[upto++] = searcherThread2; + //searcherThread2.start(); - indexerThread.join(); - indexerThread2.join(); - searcherThread1.join(); - searcherThread2.join(); + for(int i=0;i= 0) { @@ -196,7 +196,7 @@ dirName = fullDir(dirName); Directory dir = FSDirectory.open(new File(dirName)); - IndexSearcher searcher = new IndexSearcher(dir); + IndexSearcher searcher = new IndexSearcher(dir, true); IndexReader reader = searcher.getIndexReader(); _TestUtil.checkIndex(dir); @@ -232,7 +232,8 @@ // Only ID 7 is deleted assertEquals(7, i); } - + + //System.out.println("\nTEST: now search"); ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; // First document should be #21 since it's norm was @@ -275,7 +276,6 @@ // open writer IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false); - // add 10 docs for(int i=0;i<10;i++) { addDoc(writer, 35+i); @@ -292,7 +292,7 @@ writer.close(); // make sure searching sees right # hits - IndexSearcher searcher = new IndexSearcher(dir); + IndexSearcher searcher = new IndexSearcher(dir, true); ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; Document d = searcher.doc(hits[0].doc); assertEquals("wrong first document", "21", d.get("id")); @@ -301,7 +301,7 @@ // make sure we can do delete & setNorm against this // pre-lockless segment: - IndexReader reader = IndexReader.open(dir); + IndexReader reader = IndexReader.open(dir, false); Term searchTerm = new Term("id", "6"); int delCount = reader.deleteDocuments(searchTerm); assertEquals("wrong delete count", 1, delCount); @@ -309,7 +309,7 @@ reader.close(); // make sure they "took": - searcher = new IndexSearcher(dir); + searcher = new IndexSearcher(dir, true); hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; assertEquals("wrong number of hits", 43, hits.length); d = searcher.doc(hits[0].doc); @@ -322,9 +322,9 @@ writer.optimize(); writer.close(); - searcher = new IndexSearcher(dir); + searcher = new IndexSearcher(dir, true); hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; - assertEquals("wrong number of hits", 43, hits.length); + assertEquals("wrong number of hits: index=" + dirName, 43, hits.length); d = searcher.doc(hits[0].doc); testHits(hits, 43, searcher.getIndexReader()); assertEquals("wrong first document", "22", d.get("id")); @@ -342,7 +342,7 @@ Directory dir = FSDirectory.open(new File(dirName)); // make sure searching sees right # hits - IndexSearcher searcher = new IndexSearcher(dir); + IndexSearcher searcher = new IndexSearcher(dir, true); ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; assertEquals("wrong number of hits", 34, hits.length); Document d = searcher.doc(hits[0].doc); @@ -351,7 +351,7 @@ // make sure we can do a delete & setNorm against this // pre-lockless segment: - IndexReader reader = IndexReader.open(dir); + IndexReader reader = IndexReader.open(dir, false); Term searchTerm = new Term("id", "6"); int delCount = reader.deleteDocuments(searchTerm); assertEquals("wrong delete count", 1, delCount); @@ -359,7 +359,7 @@ reader.close(); // make sure they "took": - searcher = new IndexSearcher(dir); + searcher = new IndexSearcher(dir, true); hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; assertEquals("wrong number of hits", 33, hits.length); d = searcher.doc(hits[0].doc); @@ -372,9 +372,9 @@ writer.optimize(); writer.close(); - searcher = new IndexSearcher(dir); + searcher = new IndexSearcher(dir, true); hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; - assertEquals("wrong number of hits", 33, hits.length); + assertEquals("wrong number of hits; index=" + dirName, 33, hits.length); d = searcher.doc(hits[0].doc); assertEquals("wrong first document", "22", d.get("id")); testHits(hits, 33, searcher.getIndexReader()); @@ -408,7 +408,7 @@ writer.close(); // Delete one doc so we get a .del file: - IndexReader reader = IndexReader.open(dir); + IndexReader reader = IndexReader.open(dir, false); Term searchTerm = new Term("id", "7"); int delCount = reader.deleteDocuments(searchTerm); assertEquals("didn't delete the right number of documents", 1, delCount); @@ -441,7 +441,7 @@ writer.close(); // Delete one doc so we get a .del file: - IndexReader reader = IndexReader.open(dir); + IndexReader reader = IndexReader.open(dir, false); Term searchTerm = new Term("id", "7"); int delCount = reader.deleteDocuments(searchTerm); assertEquals("didn't delete the right number of documents", 1, delCount); Index: src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- src/test/org/apache/lucene/index/TestCodecs.java (revision 0) +++ src/test/org/apache/lucene/index/TestCodecs.java (revision 0) @@ -0,0 +1,562 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.*; +import org.apache.lucene.index.codecs.*; +import org.apache.lucene.index.codecs.standard.*; +import org.apache.lucene.store.*; +import java.util.*; + +// nocommit -- test multiple codecs here? + +// TODO +// - fix this test to run once for all codecs +// - make more docs per term, to test > 1 level skipping +// - test all combinations of payloads/not and omitTF/not +// - test w/ different indexDivisor +// - test field where payload length rarely changes +// - 0-term fields +// - seek/skip to same term/doc i'm already on +// - mix in deleted docs +// - seek, skip beyond end -- assert returns false +// - seek, skip to things that don't exist -- ensure it +// goes to 1 before next one known to exist +// - skipTo(term) +// - skipTo(doc) + +public class TestCodecs extends LuceneTestCase { + + // nocommit -- switch to newRandom(): + private static final Random RANDOM = new Random(42); + private static String[] fieldNames = new String[] {"one", "two", "three", "four"}; + + private final static int NUM_TEST_ITER = 4000; + // nocommit + //private final static int NUM_TEST_THREADS = 3; + private final static int NUM_TEST_THREADS = 2; + private final static int NUM_FIELDS = 4; + private final static int NUM_TERMS_RAND = 50; // must be > 16 to test skipping + private final static int DOC_FREQ_RAND = 500; // must be > 16 to test skipping + private final static int TERM_DOC_FREQ_RAND = 20; + + // start is inclusive and end is exclusive + public int nextInt(int start, int end) { + return start + RANDOM.nextInt(end-start); + } + + private int nextInt(int lim) { + return RANDOM.nextInt(lim); + } + + private boolean nextBoolean() { + return 0 == nextInt(1); + } + + char[] getRandomText() { + + final int len = 1+nextInt(10); + char[] buffer = new char[len+1]; + for(int i=0;i=0;i--) { + if (Codec.DEBUG) { + System.out.println(Thread.currentThread().getName() + ": TEST: term=" + field.terms[i].text2 + " has docFreq=" + field.terms[i].docs.length); + } + assertEquals(Thread.currentThread().getName() + ": field=" + field.fieldInfo.name + " term=" + field.terms[i].text2, TermsEnum.SeekResult.Status.FOUND, termsEnum.seek(new TermRef(field.terms[i].text2)).status); + assertEquals(field.terms[i].docs.length, termsEnum.docFreq()); + } + + // Seek to non-existent empty-string term + result = termsEnum.seek(new TermRef("")); + assertNotNull(result); + assertEquals(result.status, TermsEnum.SeekResult.Status.NOT_FOUND); + + // Make sure we're now pointing to first term + assertTrue(result.term.termEquals(new TermRef(field.terms[0].text2))); + + // Test docs enum + if (Codec.DEBUG) { + System.out.println("\nTEST: docs/positions"); + } + termsEnum.seek(new TermRef("")); + upto = 0; + do { + term = field.terms[upto]; + if (nextInt(3) == 1) { + if (Codec.DEBUG) { + System.out.println("\nTEST [" + getDesc(field, term) + "]: iterate docs..."); + } + DocsEnum docs = termsEnum.docs(null); + int upto2 = -1; + while(upto2 < term.docs.length-1) { + // Maybe skip: + final int left = term.docs.length-upto2; + int doc; + if (nextInt(3) == 1 && left >= 1) { + int inc = 1+nextInt(left-1); + upto2 += inc; + if (Codec.DEBUG) { + System.out.println("TEST [" + getDesc(field, term) + "]: skip: " + left + " docs left; skip to doc=" + term.docs[upto2] + " [" + upto2 + " of " + term.docs.length + "]"); + } + + doc = docs.advance(term.docs[upto2]); + // nocommit -- test skipping to non-existent doc + assertEquals(term.docs[upto2], doc); + } else { + doc = docs.next(); + assertTrue(doc != -1); + if (Codec.DEBUG) { + System.out.println("TEST [" + getDesc(field, term) + "]: got next doc..."); + } + upto2++; + } + assertEquals(term.docs[upto2], doc); + if (!field.omitTF) { + assertEquals(term.positions[upto2].length, docs.freq()); + if (nextInt(2) == 1) { + if (Codec.DEBUG) { + System.out.println("TEST [" + getDesc(field, term, term.docs[upto2]) + "]: check positions for doc " + term.docs[upto2] + "..."); + } + verifyPositions(term.positions[upto2], docs.positions()); + } else if (Codec.DEBUG) { + System.out.println("TEST: skip positions..."); + } + } else if (Codec.DEBUG) { + System.out.println("TEST: skip positions: omitTF=true"); + } + } + + assertEquals(DocsEnum.NO_MORE_DOCS, docs.next()); + + } else if (Codec.DEBUG) { + System.out.println("\nTEST [" + getDesc(field, term) + "]: skip docs"); + } + upto++; + + } while (termsEnum.next() != null); + + assertEquals(upto, field.terms.length); + + //termsEnum.close(); + } + } + } + + private void write(FieldInfos fieldInfos, Directory dir, FieldData[] fields) throws Throwable { + + // nocommit -- randomize this: + final int termIndexInterval = 16; + + SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, + Codecs.getDefault()); + + final FieldsConsumer consumer = state.codec.fieldsConsumer(state); + Arrays.sort(fields); + for(int i=0;i 0) { - s += "\n "; - } - s += l[i]; + public void testDeleteNullQuery() throws IOException { + Directory dir = new MockRAMDirectory(); + IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); + + for (int i = 0; i < 5; i++) { + addDoc(modifier, i, 2*i); } - return s; + + modifier.deleteDocuments(new TermQuery(new Term("nada", "nada"))); + modifier.commit(); + assertEquals(5, modifier.numDocs()); + modifier.close(); + dir.close(); } } Index: src/test/org/apache/lucene/index/TestLazyProxSkipping.java =================================================================== --- src/test/org/apache/lucene/index/TestLazyProxSkipping.java (revision 822088) +++ src/test/org/apache/lucene/index/TestLazyProxSkipping.java (working copy) @@ -47,8 +47,9 @@ private class SeekCountingDirectory extends RAMDirectory { public IndexInput openInput(String name) throws IOException { IndexInput ii = super.openInput(name); - if (name.endsWith(".prx")) { + if (name.endsWith(".prx") || name.endsWith(".pos") ) { // we decorate the proxStream with a wrapper class that allows to count the number of calls of seek() + // nocommit -- fix this: ii = new SeeksCountingStream(ii); } return ii; @@ -115,7 +116,7 @@ performTest(10); } - public void testSeek() throws IOException { + public void xxxtestSeek() throws IOException { Directory directory = new RAMDirectory(); IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); for (int i = 0; i < 10; i++) { Index: src/test/org/apache/lucene/index/TestMultiLevelSkipList.java =================================================================== --- src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (revision 822088) +++ src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (working copy) @@ -29,8 +29,9 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.MockRAMDirectory; import org.apache.lucene.util.LuceneTestCase; /** @@ -42,8 +43,18 @@ * */ public class TestMultiLevelSkipList extends LuceneTestCase { + + class CountingRAMDirectory extends MockRAMDirectory { + public IndexInput openInput(String fileName) throws IOException { + IndexInput in = super.openInput(fileName); + if (fileName.endsWith(".frq")) + in = new CountingStream(in); + return in; + } + } + public void testSimpleSkip() throws IOException { - RAMDirectory dir = new RAMDirectory(); + Directory dir = new CountingRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); Term term = new Term("test", "a"); @@ -57,9 +68,8 @@ writer.close(); IndexReader reader = SegmentReader.getOnlySegmentReader(dir); - SegmentTermPositions tp = (SegmentTermPositions) reader.termPositions(); - tp.freqStream = new CountingStream(tp.freqStream); - + TermPositions tp = reader.termPositions(); + for (int i = 0; i < 2; i++) { counter = 0; tp.seek(term); Index: src/test/org/apache/lucene/index/TestPayloads.java =================================================================== --- src/test/org/apache/lucene/index/TestPayloads.java (revision 822088) +++ src/test/org/apache/lucene/index/TestPayloads.java (working copy) @@ -38,7 +38,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.MockRAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util._TestUtil; @@ -98,7 +98,7 @@ // payload bit in the FieldInfo public void testPayloadFieldBit() throws Exception { rnd = newRandom(); - Directory ram = new RAMDirectory(); + Directory ram = new MockRAMDirectory(); PayloadAnalyzer analyzer = new PayloadAnalyzer(); IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); Document d = new Document(); @@ -154,7 +154,7 @@ public void testPayloadsEncoding() throws Exception { rnd = newRandom(); // first perform the test using a RAMDirectory - Directory dir = new RAMDirectory(); + Directory dir = new MockRAMDirectory(); performTest(dir); // now use a FSDirectory and repeat same test @@ -256,11 +256,17 @@ TermPositions tp = reader.termPositions(terms[0]); tp.next(); tp.nextPosition(); + // NOTE: prior rev of this test was failing to first + // call next here: + tp.next(); // now we don't read this payload tp.nextPosition(); assertEquals("Wrong payload length.", 1, tp.getPayloadLength()); byte[] payload = tp.getPayload(null, 0); assertEquals(payload[0], payloadData[numTerms]); + // NOTE: prior rev of this test was failing to first + // call next here: + tp.next(); tp.nextPosition(); // we don't read this payload and skip to a different document @@ -465,7 +471,7 @@ final int numDocs = 50; final ByteArrayPool pool = new ByteArrayPool(numThreads, 5); - Directory dir = new RAMDirectory(); + Directory dir = new MockRAMDirectory(); final IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); final String field = "test"; Index: src/test/org/apache/lucene/index/TestSegmentMerger.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentMerger.java (revision 822088) +++ src/test/org/apache/lucene/index/TestSegmentMerger.java (working copy) @@ -69,7 +69,8 @@ merger.closeReaders(); assertTrue(docsMerged == 2); //Should be able to open a new SegmentReader against the new directory - SegmentReader mergedReader = SegmentReader.get(new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true)); + SegmentReader mergedReader = SegmentReader.get(new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true, + -1, null, false, merger.hasProx(), merger.getCodec())); assertTrue(mergedReader != null); assertTrue(mergedReader.numDocs() == 2); Document newDoc1 = mergedReader.document(0); Index: src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentReader.java (revision 822088) +++ src/test/org/apache/lucene/index/TestSegmentReader.java (working copy) @@ -136,6 +136,9 @@ TermPositions positions = reader.termPositions(); positions.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field")); assertTrue(positions != null); + // NOTE: prior rev of this test was failing to first + // call next here: + assertTrue(positions.next()); assertTrue(positions.doc() == 0); assertTrue(positions.nextPosition() >= 0); } Index: src/test/org/apache/lucene/index/TestSegmentTermDocs.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentTermDocs.java (revision 822088) +++ src/test/org/apache/lucene/index/TestSegmentTermDocs.java (working copy) @@ -55,14 +55,13 @@ SegmentReader reader = SegmentReader.get(true, info, indexDivisor); assertTrue(reader != null); assertEquals(indexDivisor, reader.getTermInfosIndexDivisor()); - SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); - assertTrue(segTermDocs != null); - segTermDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field")); - if (segTermDocs.next() == true) - { - int docId = segTermDocs.doc(); + TermDocs termDocs = reader.termDocs(); + assertTrue(termDocs != null); + termDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field")); + if (termDocs.next() == true) { + int docId = termDocs.doc(); assertTrue(docId == 0); - int freq = segTermDocs.freq(); + int freq = termDocs.freq(); assertTrue(freq == 3); } reader.close(); @@ -77,20 +76,20 @@ //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.get(true, info, indexDivisor); assertTrue(reader != null); - SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); - assertTrue(segTermDocs != null); - segTermDocs.seek(new Term("textField2", "bad")); - assertTrue(segTermDocs.next() == false); + TermDocs termDocs = reader.termDocs(); + assertTrue(termDocs != null); + termDocs.seek(new Term("textField2", "bad")); + assertTrue(termDocs.next() == false); reader.close(); } { //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.get(true, info, indexDivisor); assertTrue(reader != null); - SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); - assertTrue(segTermDocs != null); - segTermDocs.seek(new Term("junk", "bad")); - assertTrue(segTermDocs.next() == false); + TermDocs termDocs = reader.termDocs(); + assertTrue(termDocs != null); + termDocs.seek(new Term("junk", "bad")); + assertTrue(termDocs.next() == false); reader.close(); } } Index: src/test/org/apache/lucene/index/TestSegmentTermEnum.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentTermEnum.java (revision 822088) +++ src/test/org/apache/lucene/index/TestSegmentTermEnum.java (working copy) @@ -61,23 +61,6 @@ verifyDocFreq(); } - public void testPrevTermAtEnd() throws IOException - { - Directory dir = new MockRAMDirectory(); - IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); - addDoc(writer, "aaa bbb"); - writer.close(); - SegmentReader reader = SegmentReader.getOnlySegmentReader(dir); - SegmentTermEnum termEnum = (SegmentTermEnum) reader.terms(); - assertTrue(termEnum.next()); - assertEquals("aaa", termEnum.term().text()); - assertTrue(termEnum.next()); - assertEquals("aaa", termEnum.prev().text()); - assertEquals("bbb", termEnum.term().text()); - assertFalse(termEnum.next()); - assertEquals("bbb", termEnum.prev().text()); - } - private void verifyDocFreq() throws IOException { Index: src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 822088) +++ src/test/org/apache/lucene/index/TestStressIndexing2.java (working copy) @@ -73,6 +73,7 @@ // dir1 = FSDirectory.open("foofoofoo"); Directory dir2 = new MockRAMDirectory(); // mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1); + Map docs = indexRandom(10, 100, 100, dir1); indexSerial(docs, dir2); @@ -98,8 +99,12 @@ int range=r.nextInt(20)+1; Directory dir1 = new MockRAMDirectory(); Directory dir2 = new MockRAMDirectory(); + //System.out.println("iter=" + iter + " range=" + range); + //System.out.println("TEST: index random"); Map docs = indexRandom(nThreads, iter, range, dir1); + //System.out.println("TEST: index serial"); indexSerial(docs, dir2); + //System.out.println("TEST: verify"); verifyEquals(dir1, dir2, "id"); } } @@ -176,6 +181,7 @@ public Map indexRandom(int nThreads, int iterations, int range, Directory dir) throws IOException, InterruptedException { Map docs = new HashMap(); for(int iter=0;iter<3;iter++) { + //System.out.println("TEST: iter=" + iter); IndexWriter w = new MockIndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true); w.setUseCompoundFile(false); @@ -201,7 +207,8 @@ threads[i].join(); } - // w.optimize(); + // nocommit -- comment out again + //w.optimize(); w.close(); for (int i=0; i 0) { // RuntimeException instead of IOException because // super() does not throw IOException currently: - throw new RuntimeException("MockRAMDirectory: cannot close: there are still open files: " + openFiles); + Iterator it = openFiles.values().iterator(); + System.out.println("\nMockRAMDirectory open files:"); + while(it.hasNext()) { + OpenFile openFile = (OpenFile) it.next(); + System.out.println("\nfile " + openFile.name + " opened from:\n"); + openFile.stack.printStackTrace(System.out); + } + throw new RuntimeException("MockRAMDirectory: cannot close: there are still open files"); } } Index: src/test/org/apache/lucene/store/MockRAMInputStream.java =================================================================== --- src/test/org/apache/lucene/store/MockRAMInputStream.java (revision 822088) +++ src/test/org/apache/lucene/store/MockRAMInputStream.java (working copy) @@ -44,16 +44,8 @@ // all clones get closed: if (!isClone) { synchronized(dir) { - Integer v = (Integer) dir.openFiles.get(name); - // Could be null when MockRAMDirectory.crash() was called - if (v != null) { - if (v.intValue() == 1) { - dir.openFiles.remove(name); - } else { - v = Integer.valueOf(v.intValue()-1); - dir.openFiles.put(name, v); - } - } + assert dir.openFiles.containsKey(this): "input=" + name + " is not open"; + dir.openFiles.remove(this); } } } Index: src/test/org/apache/lucene/util/pfor/TestFrameOfRef.java =================================================================== --- src/test/org/apache/lucene/util/pfor/TestFrameOfRef.java (revision 0) +++ src/test/org/apache/lucene/util/pfor/TestFrameOfRef.java (revision 0) @@ -0,0 +1,543 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* When using the Sun 1.6 jvm, the performance tests below (using doDecompPerfTest... methods) + * should be run with the -server argument to the forked jvm that is used for the + * junit tests by adding this line just before the 1st batchtest line + * in common-build.xml: + + * Using this -server may be slow for other tests, in particular for shorter tests. + */ + +import junit.framework.TestCase; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.util.Arrays; +import org.apache.lucene.util.LuceneTestCase; + + +public class TestFrameOfRef extends LuceneTestCase { + private static boolean doDecompPerfTests = true; + + private void showByte(int b, StringBuffer buf) { // for debugging + for (int i = 7; i >= 0; i--) { + buf.append((b >>> i) & 1); + } + } + + private void showBytes(byte[] array) { // for debugging + StringBuffer buf = new StringBuffer(); + for (int i = 0; i < array.length; i++) { + showByte(array[i] & 255, buf); + if (((i+1) % 4) != 0) { + buf.append(' '); + } else { + System.out.println(buf); + buf.setLength(0); + } + } + } + + /** Run compression without buffer, return the IntBuffer size needed for compression. */ + static int doNoBufferRun(int[] input, int offset, int numFrameBits, FrameOfRef frameOfRef) { + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.compress(numFrameBits); + return frameOfRef.compressedSize(); + } + + FrameOfRef forNoBufferCompress = new FrameOfRef(); + + /** Create an IntBuffer, compress the given input into this buffer, and return it. */ + static IntBuffer compressToBuffer( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + FrameOfRef frameOfRef) { + // Allocate an IntBuffer as a view on a ByteBuffer + ByteBuffer byteBuffer = ByteBuffer.allocate(4 * compressBufferSize); + assert byteBuffer.hasArray(); + byte[] bufferByteArray = byteBuffer.array(); + assert bufferByteArray != null; + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + + // Compress to buffer: + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.setCompressedBuffer(compressBuffer); + frameOfRef.compress(numFrameBits); +// assert bufferByteArray.length == 4 * compressBufferSize; // for showBytes() below. +// showBytes(bufferByteArray); + if (compressBufferSize >= 0) { + //assertEquals("IntBuffer size after compress() to buffer", compressBufferSize, pforCompress.compressedSize()); + } + return compressBuffer; + } + + static void deCompressFromBufferVerify( + IntBuffer compressBuffer, + int[] input, + int offset, + int compressBufferSize, + FrameOfRef frameOfRef) { + // Decompress from the buffer: + frameOfRef.setCompressedBuffer(compressBuffer); + assertEquals("Decompressed length before decompress()", input.length - offset, frameOfRef.decompressedSize()); + int[] output = new int[input.length]; // use same offset as input + frameOfRef.setUnCompressedData(output, offset, output.length - offset); + frameOfRef.decompress(); + assertEquals("IntBuffer size after decompress()", compressBufferSize, frameOfRef.compressedSize()); + if (! Arrays.equals(input, output)) { + for (int i = 0; i < input.length; i++) { + System.out.print("at index " + i + " output " + output[i]); + System.out.print((input[i] != output[i]) ? " !=" : " =="); + System.out.println(" input " + input[i]); + } + assertEquals("equal array lengths", input.length, output.length); + assertTrue("input == output", Arrays.equals(input, output)); + } + } + + private static void doTestOffset( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + String testName) { +System.out.println(); +System.out.println(testName); + FrameOfRef frameOfRef = new FrameOfRef(); + int actIntBufferSize = doNoBufferRun(input, offset, numFrameBits, frameOfRef); + assertEquals("IntBuffer size after noBuffer run compress()", compressBufferSize, actIntBufferSize); + FrameOfRef frameOfRef2 = new FrameOfRef(); + IntBuffer compressBuffer = compressToBuffer(input, offset, numFrameBits, actIntBufferSize, frameOfRef2); + // Decompress and verify against original input. + FrameOfRef frameOfRef3 = new FrameOfRef(); + deCompressFromBufferVerify(compressBuffer, input, offset, actIntBufferSize, frameOfRef3); + } + + private void doTest(int[] input, int numBits, int compressBufferSize) { + int offset = 0; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test01NoExc() { + int[] input = {1}; // no exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test07_Offset1() { + int[] input = {0,1}; + int offset = 1; + int numBits = 1; + int compressBufferSize = 2; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test07_Offset2() { + int[] input = new int[10]; + int offset = 9; + input[offset] = 1; + int numBits = 1; + int compressBufferSize = 2; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test12Series8Base4() { + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 4; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test13Series8Base5() { + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 5; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + private void frameBitsForCompressionTest(int[] input, int expectedNumFrameBits) { +System.out.println(); +System.out.println(getName()); + FrameOfRef frameOfRef = new FrameOfRef(); + frameOfRef.setUnCompressedData(input, 0, input.length); + assertEquals("numFrameBits", expectedNumFrameBits, frameOfRef.frameBitsForCompression()); + } + + public void test20frameBitsForCompression() { + int[] input = {2}; + frameBitsForCompressionTest(input, 2); + } + + public void test21frameBitsForCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 4); + } + + static void noBufferCompressionTest(int[] input, FrameOfRef frameOfRef, String testName) { +System.out.println(); +System.out.println(testName); + // Run compression without buffer: + final int offset = 0; + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.compress(); +System.out.println("Compress w/o buffer " + input.length + " ints into " + + frameOfRef.compressedSize() + + ", ratio " + (frameOfRef.compressedSize()/(float)input.length)); + } + + public void test30NoBufferCompression() { + int[] input = {0,1,0,1,0,1,0,70000}; // would force exceptions for numFrameBits == 1 + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test31NoBufferCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454}; + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test32NoBufferCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454, + 9,8,7,6,5,4,3,2,1,0,0}; + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test40For1Decompress() { + int[] input = { + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0}; + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test41For2Decompress() { + int[] input = { + 1,0,3,2,2,3,0,1, + 1,0,3,2,2,3,0,1, + 1,0,3,2,2,3,0,1, + 1,0,3,2}; + int numBits = 2; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test42For3Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 7,5,4,5,6,7,0,1, + 1,0,3,6,4,7,5,1, + 1,0,4,5,6,7,0,1, // 32 input, 3 ints compressed + 1,0,4,5,6,7,0,1, + 4,6,3,6,4,7,5,1, + 1,0,4,5,6,7}; // 22 more input, 9 bytes compressed + int numBits = 3; + int compressBufferSize = 7; + doTest(input, numBits, compressBufferSize); + } + + public void test43For4Decompress() { + int[] input = { + 1,0,3,2,5,7,4,6, + 8,9,10,2,15,0}; + int numBits = 4; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test447For17Decompress() { + int[] input = {1,1022,1023}; + int numBits = 17; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test449For17Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 17; + int compressBufferSize = 18; + doTest(input, numBits, compressBufferSize); + } + + /** Repeat decompressing from a given buffer, report on performance. */ + static void doDecompPerfTestUsing( + IntBuffer comprDataBuffer, + int decomprSize, + FrameOfRef frameOfRef, + String testName) { + frameOfRef.setCompressedBuffer(comprDataBuffer); + for (int rep = 0; rep < 3; rep++) { + long maxTestMillis = 300; // 1000 + long testMillis; + int iterations = 0; + long startMillis = System.currentTimeMillis(); + final int decompsPerIter = 1024 * 128; + int[] output = new int[decomprSize]; // use 0 offset + frameOfRef.setUnCompressedData(output, 0, decomprSize); // FIXME: use decomprSize from compressBuffer, or even from pforDecompress + do { + for (int i = 0; i < decompsPerIter; i++) { + frameOfRef.decompress(); + } + iterations++; + testMillis = System.currentTimeMillis() - startMillis; + } while ((testMillis < maxTestMillis) && (iterations < 1000)); + long totalDecompressed = (((long) decomprSize) * decompsPerIter * iterations); + System.out.println(testName + " " + rep + + " bits " + frameOfRef.getNumFrameBits() + + " decompressed " + totalDecompressed + + " in " + testMillis + " msecs, " + + ((int)(totalDecompressed/(testMillis * 1000f))) + " kints/msec, (" + + iterations + " iters)."); + } + } + + + private void doDecompPerfTest(IntBuffer comprDataBuffer, int decomprSize) { + if (! doDecompPerfTests) { + return; + } + FrameOfRef frameOfRef = new FrameOfRef(); + doDecompPerfTestUsing(comprDataBuffer, decomprSize, frameOfRef, getName()); + } + + private void doCompDecompPerfTest(int[] decomprData, int numBits) { + int offset = 0; + FrameOfRef frameOfRef = new FrameOfRef(); + int actIntBufferSize = doNoBufferRun(decomprData, offset, numBits, frameOfRef); + FrameOfRef frameOfRef2 = new FrameOfRef(); + IntBuffer comprDataBuffer = compressToBuffer(decomprData, offset, numBits, actIntBufferSize, frameOfRef2); + if (! doDecompPerfTests) { + return; + } + FrameOfRef frameOfRef3 = new FrameOfRef(); + doDecompPerfTestUsing(comprDataBuffer, decomprData.length, frameOfRef3, getName()); + } + + public void test91PerfFor01Decompress() { + int[] input = { + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0}; + int numBits = 1; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor02Decompress() { + int[] input = { + 1,0,3,2,3,2,1,0, + 1,0,3,2,3,2,1,0, + 1,0,3,2,3,1,0,1, + 1,0,3,2,3,2,1,0}; + int numBits = 2; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor03Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 7,5,4,5,6,7,0,1, + 1,0,3,6,4,7,5,1, + 1,0,4,5,6,7,0,1}; + int numBits = 3; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor04Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 9,8,11,14,12,15,13,9, + 7,5,4,5,6,7,0,1, + 9,8,11,14,12,15,13,9}; + int numBits = 4; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor05Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 9,8,11,14,12,15,13,9, + 23,21,20,21,22,23,16,17, + 9,8,11,14,12,15,13,9}; + int numBits = 5; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor06Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 33,32,35,37,63,62,61,60, + 9,8,11,14,12,15,13,9, + 23,21,20,21,22,23,16,17}; + int numBits = 6; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor07Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 33,32,35,37,63,62,61,60, + 9,8,11,14,127,126,13,9, + 23,21,20,21,22,23,16,17}; + int numBits = 7; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor08Decompress() { + int[] input = { + 1,0,3,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,127,126,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 8; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor09Decompress() { + int[] input = { + 1,0,3,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 9; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor10Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 10; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor11Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 11; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor12Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 12; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor13Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 13; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor14Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 14; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor15Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32766,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 15; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor16Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65534,65535}; + int numBits = 16; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor17Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 17; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor18Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,131071,131072,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 18; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor19Decompress() { + int[] input = { + 1,1022,1023,127,262144,262143,4,7, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,131071,131072,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 19; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor20_32Decompress() { + for (int numBits = 20; numBits <= 32; numBits++) { + int[] input = { + 1,(int)((1L< + * Using this -server may be slow for other tests, in particular for shorter tests. + */ + +import junit.framework.TestCase; +import java.nio.IntBuffer; +import org.apache.lucene.util.LuceneTestCase; + +/** + * To be added: performance tests for decoding exceptions. + */ +public class TestPFor extends LuceneTestCase { + private static boolean doDecompPerfTests = true; + + private static void doTestOffset( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + String testName) { +System.out.println(); +System.out.println(testName); + PFor pFor = new PFor(); + int actIntBufferSize = TestFrameOfRef.doNoBufferRun(input, offset, numFrameBits, pFor); + assertEquals("IntBuffer size after noBuffer run compress()", compressBufferSize, actIntBufferSize); + PFor pFor2 = new PFor(); + IntBuffer compressBuffer = TestFrameOfRef.compressToBuffer(input, offset, numFrameBits, actIntBufferSize, pFor2); + // Decompress and verify against original input. + PFor pFor3 = new PFor(); + TestFrameOfRef.deCompressFromBufferVerify(compressBuffer, input, offset, actIntBufferSize, pFor3); + } + + private void doTest(int[] input, int numBits, int compressBufferSize) { + int offset = 0; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test02ExcByte1() { + int[] input = {2}; // 1 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte2() { + int[] input = {1,2}; // 1 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte3() { + int[] input = {1,(1<<7)}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte4() { + int[] input = {1,(1<<7),0}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte5() { + int[] input = {1,(1<<7),0,65}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte1() { + int[] input = {1<<8}; // 2 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte2() { + int[] input = {1<<8, 1}; // 2 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte3() { + int[] input = {1<<8, 1, 2}; // 2 byte exception + int numBits = 3; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte4() { + int[] input = {1<<8, 1, 2}; // 2 byte exception + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte5() { + int[] input = {1<<8, 1, 1<<9}; // two 2 byte exceptions + int numBits = 2; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte6() { + int[] input = {1<<8, 1, 1<<9}; // two 2 byte exceptions + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test05ExcThreeByte() { + int[] input = {1<<16}; // 4 byte exception + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte1() { + int[] input = {1<<30}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte2() { + int[] input = {1<<30,0}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte3() { + int[] input = {1,1<<30,0}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test08ForcedException1() { + int[] input = {2,1,1}; // 2 exceptions, 1 byte + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test09ForcedException2() { + int[] input = {(1<<24),1,0}; // 2 exceptions, 4 byte + int numBits = 1; + int compressBufferSize = 4; + doTest(input, numBits, compressBufferSize); + } + + public void test10FirstException() { + int[] input = {0,1,2,3,0,1,6,7,8}; // Test for not forcing first exception at index 4 (2nd value 0) + int numBits = 2; + int compressBufferSize = 3; // 3 exceptions from value 6 + doTest(input, numBits, compressBufferSize); + } + + public void test11Series8Base3() { // This also tests for not forcing first exception + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 3; + int compressBufferSize = 3; // 1 exception + doTest(input, numBits, compressBufferSize); + } + + + private void frameBitsForCompressionTest(int[] input, int expectedNumFrameBits) { +System.out.println(); +System.out.println(getName()); + PFor pfor = new PFor(); + pfor.setUnCompressedData(input, 0, input.length); + assertEquals("numFrameBits", expectedNumFrameBits, pfor.frameBitsForCompression()); + } + + public void test20frameBitsForCompression() { + int[] input = {2}; + frameBitsForCompressionTest(input, 2); + } + + public void test21frameBitsForCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 4); + } + + public void test22frameBitsForCompression() { + int[] input = {16000,16001,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 3); + } + + private void doDecompPerfTest(int[] decomprData, int numBitsNoExceptions) { + doDecompPerfTestNoExceptions(decomprData, numBitsNoExceptions); + for (int nb = numBitsNoExceptions-1; nb >= numBitsNoExceptions-3; nb--) { + if (nb <= 0) + break; + doDecompPerfTestExceptions(decomprData, nb); + } + } + + private void checkExceptions(int inputLength, int numBits, int compressBufferSize, boolean exceptionsPresent) { + int sizeNoExceptions = 1 + (inputLength * numBits + 31) / 32; + if (exceptionsPresent) { + assertTrue(("Performance test with exceptions, IntBuffer " + sizeNoExceptions + " < " + compressBufferSize), + sizeNoExceptions < compressBufferSize); + System.out.println("Nr of exception quadbytes: " + (compressBufferSize - sizeNoExceptions)); + } else { + assertEquals("Performance test without exceptions, IntBuffer size", sizeNoExceptions, compressBufferSize); + } + } + + private void doDecompPerfTestNoExceptions(int[] decomprData, int numBits) { + assert decomprData.length % 32 == 0 : decomprData.length; + PFor pFor = new PFor(); + int comprDataSize = TestFrameOfRef.doNoBufferRun(decomprData, 0, numBits, pFor); + checkExceptions(decomprData.length, numBits, comprDataSize, false); + IntBuffer comprDataBuffer = TestFrameOfRef.compressToBuffer(decomprData, 0, numBits, comprDataSize, pFor); + // Verify that decompression is correct: + TestFrameOfRef.deCompressFromBufferVerify(comprDataBuffer, decomprData, 0, comprDataSize, pFor); + + if (! doDecompPerfTests) { + return; + } +System.out.println(); +System.out.println(getName() + " starting, numFrameBits " + numBits); + doDecompPerfTest(comprDataBuffer, decomprData.length); + } + + private void doDecompPerfTestExceptions(int[] decomprData, int numBits) { + assert decomprData.length % 32 == 0 : decomprData.length; + PFor pFor = new PFor(); + int comprDataSize = TestFrameOfRef.doNoBufferRun(decomprData, 0, numBits, pFor); + checkExceptions(decomprData.length, numBits, comprDataSize, true); + IntBuffer comprDataBuffer = TestFrameOfRef.compressToBuffer(decomprData, 0, numBits, comprDataSize, pFor); + // Verify that decompression is correct: + TestFrameOfRef.deCompressFromBufferVerify(comprDataBuffer, decomprData, 0, comprDataSize, pFor); + + if (! doDecompPerfTests) { + return; + } +System.out.println(); +System.out.println(getName() + " starting, numFrameBits " + numBits); + doDecompPerfTest(comprDataBuffer, decomprData.length); + } + + + private void doDecompPerfTest(IntBuffer comprDataBuffer, int decomprSize) { + TestFrameOfRef.doDecompPerfTestUsing(comprDataBuffer, decomprSize, new PFor(), getName()); + } + + public void tst92PerfPFor20_32Decompress() { + for (int numBits = 20; numBits <= 32; numBits++) { + int[] input = { + 1,(int)((1L< \n"); + System.out.println("Eg: java org.apache.lucene.util.TestPFor2 /lucene/index _l.prx _l.prx.pfor\n"); + System.exit(1); + } + + String vIntFileNameIn = args[1]; + String pForFileNameOut = args[2]; + + // Convert vInt encoding --> pfor + if (!dir.fileExists(pForFileNameOut)) { + System.out.println("\nencode " + vIntFileNameIn + " to " + pForFileNameOut + "..."); + convertVIntToPFor(dir, vIntFileNameIn, pForFileNameOut); + } + + System.out.println("\ndecompress using pfor:"); + long bestPFor = 0; + for(int round=0;round<5;round++) { + long speed = readPFor(dir, pForFileNameOut); + if (speed > bestPFor) + bestPFor = speed; + } + + System.out.println("\ndecompress using readVInt:"); + long bestVInt = 0; + for(int round=0;round<5;round++) { + long speed = readVInts(dir, vIntFileNameIn); + if (speed > bestVInt) + bestVInt = speed; + } + + NumberFormat nf = NumberFormat.getInstance(); + if (bestVInt > bestPFor) + System.out.println("\nPFor is " + nf.format((bestVInt-bestPFor)*100.0/bestVInt) + "% slower"); + else + System.out.println("\nPFor is " + nf.format((bestPFor-bestVInt)*100.0/bestVInt) + "% faster"); + + dir.close(); + } + + /** Returns ints/sec speed */ + public static long readVInts(Directory dir, String vIntFileNameIn) throws Throwable { + IndexInput in = dir.openInput(vIntFileNameIn); + final long t0 = System.currentTimeMillis(); + long count = 0; + while(true) { + try { + in.readVInt(); + count++; + } catch (IOException ioe) { + break; + } + } + final long t1 = System.currentTimeMillis(); + in.close(); + System.out.println((t1-t0) + " msec to read " + count + " ints (" + (count/(t1-t0)) + " ints/msec)"); + + return count/(t1-t0); + } + + /** Returns ints/sec speed */ + public static long readPFor(Directory dir, String pForFileNameOut) throws Throwable { + IndexInput in = dir.openInput(pForFileNameOut); + + PFor pforDecompress = new PFor(); + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + byte[] bufferByteArray = byteBuffer.array(); + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + pforDecompress.setCompressedBuffer(compressBuffer); + final int[] temp = new int[BLOCK_SIZE]; + final long t0 = System.currentTimeMillis(); + long count = 0; + while(true) { + try { + int numByte = in.readInt(); + in.readBytes(bufferByteArray, 0, numByte); + pforDecompress.setUnCompressedData(temp, 0, 0); + pforDecompress.decompress(); + count++; + } catch (IOException ioe) { + break; + } + } + final long t1 = System.currentTimeMillis(); + System.out.println((t1-t0) + " msec to decode " + (BLOCK_SIZE*count) + " ints (" + (BLOCK_SIZE*count/(t1-t0)) + " ints/msec)"); + in.close(); + + return (BLOCK_SIZE*count)/(t1-t0); + } + + // nocommit -- so JUnit doesn't complain + public void test() { + assertTrue(true); + } + + public static void convertVIntToPFor(Directory dir, String vIntFileNameIn, String pForFileNameOut) throws Throwable { + IndexInput in = dir.openInput(vIntFileNameIn); + IndexOutput out = dir.createOutput(pForFileNameOut); + + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + byte[] bufferByteArray = byteBuffer.array(); + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + + PFor pforCompress = new PFor(); + pforCompress.setCompressedBuffer(compressBuffer); + + // Get ints + int count = 0; + int upto = 0; + int[] temp = new int[BLOCK_SIZE]; + + final Random r = new Random(); + final int[] counts = new int[32]; + + while(true) { + try { + temp[upto++] = in.readVInt(); + } catch (IOException ioe) { + break; + } + if (upto == BLOCK_SIZE) { + pforCompress.setUnCompressedData(temp, 0, BLOCK_SIZE); + final int numFrameBits = pforCompress.frameBitsForCompression(); + counts[numFrameBits]++; + pforCompress.compress(); + final int numByte = pforCompress.compressedSize() * 4; + out.writeInt(numByte); + out.writeBytes(bufferByteArray, 0, numByte); + upto = 0; + count++; + } + } + in.close(); + out.close(); + System.out.println((BLOCK_SIZE*count) + " ints; " + dir.fileLength(pForFileNameOut) + " bytes compressed vs orig size " + dir.fileLength(vIntFileNameIn)); + + /* + NumberFormat nf = NumberFormat.getInstance(); + for(int i=1;i<31;i++) + System.out.println(i + " bits: " + counts[i] + " [" + nf.format(100.0*counts[i]/count) + " %]"); + */ + } +} \ No newline at end of file