Index: contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java =================================================================== --- contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 803321) +++ contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy) @@ -203,7 +203,7 @@ private Analyzer analyzer; private boolean fastMode = false; - private final boolean verbose = false; + private final boolean verbose = true; private static final String FIELD_NAME = "content"; @@ -333,7 +333,7 @@ if (useMemIndex && useRAMIndex) { if (verbose) System.out.println("diff="+ (score1-score2) + ", query=" + queries[q] + ", s1=" + score1 + ", s2=" + score2); if (score1 != score2 || score1 < 0.0f || score2 < 0.0f || score1 > 1.0f || score2 > 1.0f) { - throw new IllegalStateException("BUG DETECTED:" + (i*(q+1)) + " at query=" + queries[q] + ", file=" + file + ", anal=" + analyzer); + throw new IllegalStateException("BUG DETECTED:" + (i*(q+1)) + " at query=" + queries[q] + ", file=" + file + ", anal=" + analyzer + " score1=" + score1 + " score2=" + score2); } } } @@ -424,9 +424,11 @@ else searcher = ((MemoryIndex) index).createSearcher(); + System.out.println("now search"); final float[] scores = new float[1]; // inits to 0.0f searcher.search(query, new HitCollector() { public void collect(int doc, float score) { + System.out.println(" collect score=" + score); scores[0] = score; } }); Index: src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- src/java/org/apache/lucene/index/CheckIndex.java (revision 803321) +++ src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -269,6 +269,7 @@ infoStream.println(msg); } + /* private static class MySegmentTermDocs extends SegmentTermDocs { int delCount; @@ -286,7 +287,8 @@ delCount++; } } - + */ + /** Returns true if index is clean, else false. * @deprecated Please instantiate a CheckIndex and then use {@link #checkIndex()} instead */ public static boolean check(Directory dir, boolean doFix) throws IOException { @@ -317,6 +319,10 @@ return checkIndex(null); } + protected Status checkIndex(List onlySegments) throws IOException { + return checkIndex(onlySegments, PostingsCodecs.getDefault()); + } + /** Returns a {@link Status} instance detailing * the state of the index. * @@ -329,13 +335,13 @@ *

WARNING: make sure * you only call this when the index is not opened by any * writer. */ - public Status checkIndex(List onlySegments) throws IOException { + protected Status checkIndex(List onlySegments, PostingsCodecs codecs) throws IOException { NumberFormat nf = NumberFormat.getInstance(); SegmentInfos sis = new SegmentInfos(); Status result = new Status(); result.dir = dir; try { - sis.read(dir); + sis.read(dir, codecs); } catch (Throwable t) { msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; @@ -392,6 +398,8 @@ sFormat = "FORMAT_USER_DATA [Lucene 2.9]"; else if (format == SegmentInfos.FORMAT_DIAGNOSTICS) sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]"; + else if (format == SegmentInfos.FORMAT_FLEX_POSTINGS) + sFormat = "FORMAT_FLEX_POSTINGS [Lucene 2.9]"; else if (format < SegmentInfos.CURRENT_FORMAT) { sFormat = "int=" + format + " [newer version of Lucene than this tool]"; skip = true; @@ -622,7 +630,14 @@ final TermPositions termPositions = reader.termPositions(); // Used only to count up # deleted docs for this term - final MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader); + /* + // nocommit -- make this work w/ flexi too + final MySegmentTermDocs myTermDocs; + if (info.getFlexPostings()) + myTermDocs = null; + else + myTermDocs = new MySegmentTermDocs(reader); + */ final int maxDoc = reader.maxDoc(); @@ -660,19 +675,24 @@ // Now count how many deleted docs occurred in // this term: - final int delCount; - if (reader.hasDeletions()) { - myTermDocs.seek(term); - while(myTermDocs.next()) { } - delCount = myTermDocs.delCount; - } else { - delCount = 0; - } + + // nocommit -- do this check w/ flex postings too + /* + if (!info.getFlexPostings()) { + final int delCount = 0; } + if (reader.hasDeletions()) { + myTermDocs.seek(term); + while(myTermDocs.next()) { + } + delCount = myTermDocs.delCount; + } else + delCount = 0; - if (freq0 + delCount != docFreq) { - throw new RuntimeException("term " + term + " docFreq=" + - docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount); - } + if (freq0 + delCount != docFreq) + throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount); + } + */ + } msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]"); Index: src/java/org/apache/lucene/index/DefaultCodec.java =================================================================== --- src/java/org/apache/lucene/index/DefaultCodec.java (revision 0) +++ src/java/org/apache/lucene/index/DefaultCodec.java (revision 0) @@ -0,0 +1,66 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.io.IOException; + +import org.apache.lucene.store.Directory; + +/** Current index file format */ +class DefaultCodec extends PostingsCodec { + + // nocommit -- give it better name than Default -- the + // fact that it'sd default should be known elsewhere + DefaultCodec() { + name = "Default"; + } + + FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + DocsConsumer docs = new FormatPostingsDocsWriter(state); + boolean success = false; + try { + FieldsConsumer ret = new FormatPostingsTermsDictWriter(state, docs); + success = true; + return ret; + } finally { + if (!success) + docs.close(); + } + } + + FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException { + FormatPostingsTermsDictDocsReader docs = new FormatPostingsDocsReader(dir, si, readBufferSize); + boolean success = false; + try { + FieldsProducer ret = new FormatPostingsTermsDictReader(dir, fieldInfos, si.name, + docs, + readBufferSize, indexDivisor); + success = true; + return ret; + } finally { + if (!success) + docs.close(); + } + } + + void files(SegmentInfo segmentInfo, Collection files) { + FormatPostingsDocsReader.files(segmentInfo, files); + FormatPostingsTermsDictReader.files(segmentInfo, files); + } +} Index: src/java/org/apache/lucene/index/DefaultSkipListWriter.java =================================================================== --- src/java/org/apache/lucene/index/DefaultSkipListWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/DefaultSkipListWriter.java (working copy) @@ -35,7 +35,8 @@ private long[] lastSkipProxPointer; private IndexOutput freqOutput; - private IndexOutput proxOutput; + // nocommit -- private again + IndexOutput proxOutput; private int curDoc; private boolean curStorePayloads; @@ -81,6 +82,12 @@ Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); if (proxOutput != null) Arrays.fill(lastSkipProxPointer, proxOutput.getFilePointer()); + if (PostingsCodec.DEBUG) { + if (proxOutput != null) + System.out.println(" skip writer base freqFP=" + freqOutput.getFilePointer() + " proxFP=" + proxOutput.getFilePointer()); + else + System.out.println(" skip writer base freqFP=" + freqOutput.getFilePointer()); + } } protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { Index: src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- src/java/org/apache/lucene/index/DirectoryReader.java (revision 803321) +++ src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -17,25 +17,25 @@ * limitations under the License. */ +import java.io.FileNotFoundException; import java.io.IOException; -import java.io.FileNotFoundException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Set; -import java.util.Collections; -import java.util.ArrayList; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.store.AlreadyClosedException; /** * An IndexReader which reads indexes with multiple segments. @@ -43,6 +43,8 @@ class DirectoryReader extends IndexReader implements Cloneable { protected Directory directory; protected boolean readOnly; + + protected PostingsCodecs codecs; IndexWriter writer; @@ -63,28 +65,48 @@ private int numDocs = -1; private boolean hasDeletions = false; +// static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, +// final int termInfosIndexDivisor) throws CorruptIndexException, IOException { +// return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); +// } + static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, - final int termInfosIndexDivisor) throws CorruptIndexException, IOException { + final int termInfosIndexDivisor, PostingsCodecs codecs) throws CorruptIndexException, IOException { + final PostingsCodecs codecs2; + if (codecs == null) + codecs2 = PostingsCodecs.getDefault(); + else + codecs2 = codecs; return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { SegmentInfos infos = new SegmentInfos(); - infos.read(directory, segmentFileName); + infos.read(directory, segmentFileName, codecs2); if (readOnly) - return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor); + return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, codecs2); else - return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor); + return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, codecs2); } }.run(commit); } /** Construct reading the named set of readers. */ - DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { +// DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { +// this(directory, sis, deletionPolicy, readOnly, termInfosIndexDivisor, null); +// } + + /** Construct reading the named set of readers. */ + DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, PostingsCodecs codecs) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = sis; this.deletionPolicy = deletionPolicy; this.termInfosIndexDivisor = termInfosIndexDivisor; + if (codecs == null) + this.codecs = PostingsCodecs.getDefault(); + else + this.codecs = codecs; + if (!readOnly) { // We assume that this segments_N was previously // properly sync'd: @@ -120,11 +142,17 @@ } // Used by near real-time search - DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException { + DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, PostingsCodecs codecs) throws IOException { this.directory = writer.getDirectory(); this.readOnly = true; this.segmentInfos = infos; this.termInfosIndexDivisor = termInfosIndexDivisor; + if (codecs == null) + this.codecs = PostingsCodecs.getDefault(); + else + this.codecs = codecs; + + if (!readOnly) { // We assume that this segments_N was previously // properly sync'd: @@ -175,11 +203,17 @@ /** This contructor is only used for {@link #reopen()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, - Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException { + Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, PostingsCodecs codecs) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = infos; this.termInfosIndexDivisor = termInfosIndexDivisor; + if (codecs == null) + this.codecs = PostingsCodecs.getDefault(); + else + this.codecs = codecs; + + if (!readOnly) { // We assume that this segments_N was previously // properly sync'd: @@ -423,7 +457,7 @@ return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { SegmentInfos infos = new SegmentInfos(); - infos.read(directory, segmentFileName); + infos.read(directory, segmentFileName, codecs); return doReopen(infos, false, openReadOnly); } }.run(commit); @@ -432,9 +466,9 @@ private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { DirectoryReader reader; if (openReadOnly) { - reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor); + reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, null); } else { - reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor); + reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, null); } reader.setDisableFakeNorms(getDisableFakeNorms()); return reader; @@ -630,6 +664,10 @@ ensureOpen(); return new MultiTermDocs(this, subReaders, starts); } + + public FieldsEnum fields() throws IOException { + return new MultiFieldsEnum(subReaders, starts); + } public TermPositions termPositions() throws IOException { ensureOpen(); @@ -669,7 +707,7 @@ // we have to check whether index has changed since this reader was opened. // if so, this reader is no longer valid for deletion - if (SegmentInfos.readCurrentVersion(directory) > segmentInfos.getVersion()) { + if (SegmentInfos.readCurrentVersion(directory, codecs) > segmentInfos.getVersion()) { stale = true; this.writeLock.release(); this.writeLock = null; @@ -699,7 +737,7 @@ // KeepOnlyLastCommitDeleter: IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, null, null); + segmentInfos, null, null, codecs); // Checkpoint the state we are about to change, in // case we have to roll back: @@ -794,7 +832,7 @@ */ public boolean isCurrent() throws CorruptIndexException, IOException { ensureOpen(); - return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion(); + return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion(); } protected synchronized void doClose() throws IOException { @@ -861,12 +899,17 @@ /** @see org.apache.lucene.index.IndexReader#listCommits */ public static Collection listCommits(Directory dir) throws IOException { + return listCommits(dir, PostingsCodecs.getDefault()); + } + + /** @see org.apache.lucene.index.IndexReader#listCommits */ + public static Collection listCommits(Directory dir, PostingsCodecs codecs) throws IOException { final String[] files = dir.listAll(); Collection commits = new ArrayList(); SegmentInfos latest = new SegmentInfos(); - latest.read(dir); + latest.read(dir, codecs); final long currentGen = latest.getGeneration(); commits.add(new ReaderCommit(latest, dir)); @@ -883,7 +926,7 @@ try { // IOException allowed to throw there, in case // segments_N is corrupt - sis.read(dir, fileName); + sis.read(dir, fileName, codecs); } catch (FileNotFoundException fnfe) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -954,7 +997,269 @@ return userData; } } + + static class FieldPQData { + final FieldsEnum fields; + TermsEnum terms; + final int start; + final IndexReader reader; + FieldPQData(FieldsEnum fields, int start, IndexReader reader) { + this.fields = fields; + this.start = start; + this.reader = reader; + } + + void close() throws IOException { + //fields.close(); + } + } + + private static class MultiFieldsEnum extends FieldsEnum { + private String currentField; + private final IndexReader[] readers; + private final int[] starts; + + private final FieldPQData[] fields; + + private final FieldPQData[] sameFields; + int numSameFields; + + MultiFieldsEnum(IndexReader[] readers, int[] starts) throws IOException { + this.readers = readers; + this.starts = starts; + fields = new FieldPQData[readers.length]; + for(int i=0;i 0) { + FieldPQData top = (FieldPQData) queue.top(); + currentField = top.fields.field(); + return true; + } else { + return false; + } + } + */ + + public boolean next() { + // nocommit -- todo + throw new UnsupportedOperationException(); + } + + public boolean seek(String field) throws IOException { + boolean result; + currentField = field; + numSameFields = 0; + for(int i=0;i 0; + } + + private final MultiTermsEnum terms = new MultiTermsEnum(); + + public TermsEnum terms() throws IOException { + terms.reset(sameFields, numSameFields); + return terms; + } + + public void close() { + }; + } + + private static class MultiTermsEnum extends TermsEnum { + + FieldPQData[] fields; + int numFields; + + FieldPQData[] sameText; + int numSame; + + void reset(FieldPQData[] fields, int numFields) throws IOException { + this.fields = fields; + this.numFields = numFields; + for(int i=0;i 0; + } + + public String text() { + throw new UnsupportedOperationException(); + } + + public boolean next() { + // nocommit todo + throw new UnsupportedOperationException(); + } + + public void close() { + } + + public int docFreq() { + int sum = 0; + for(int i=0;i= target */ + public abstract int skipTo(int target) throws IOException; + + /** Returns the next docID, or -1 at the end. */ + public abstract int next() throws IOException; + + public abstract int freq(); + + public abstract int ord(); + + /** Bulk read: returns number of docs read. */ + public abstract int read(int[] docs, int[] freqs) throws IOException; + + // nocommit -- maybe move this up to TermsEnum? that + // would disallow changing positions format/reader of each + // doc, though + /** Don't call next() or skipTo() or read() until you're + * done consuming the positions */ + public abstract PositionsEnum positions() throws IOException; + + //public abstract void close() throws IOException; +} Index: src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -543,9 +543,16 @@ synchronized private void initFlushState(boolean onlyDocStore) { initSegmentName(onlyDocStore); - flushState = new SegmentWriteState(this, directory, segment, docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval()); + flushState = new SegmentWriteState(this, directory, segment, docFieldProcessor.fieldInfos, + docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval(), + writer.codecs); } + /** Returns the codec used to flush the last segment */ + PostingsCodec getCodec() { + return flushState.codec; + } + /** Flush all pending docs to a new segment */ synchronized int flush(boolean closeDocStore) throws IOException { @@ -611,8 +618,12 @@ CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION); Iterator it = flushState.flushedFiles.iterator(); - while(it.hasNext()) - cfsWriter.addFile((String) it.next()); + while(it.hasNext()) { + final String fileName = (String) it.next(); + if (PostingsCodec.DEBUG) + System.out.println("make cfs " + fileName); + cfsWriter.addFile(fileName); + } // Perform the merge cfsWriter.close(); @@ -968,24 +979,24 @@ // Delete by term Iterator iter = deletesFlushed.terms.entrySet().iterator(); - TermDocs docs = reader.termDocs(); + try { while (iter.hasNext()) { Entry entry = (Entry) iter.next(); Term term = (Term) entry.getKey(); - docs.seek(term); + DocsEnum docs = reader.termDocsEnum(term); int limit = ((BufferedDeletes.Num) entry.getValue()).getNum(); - while (docs.next()) { - int docID = docs.doc(); - if (docIDStart+docID >= limit) - break; + while (true) { + final int docID = docs.next(); + if (docID == -1 || docIDStart+docID >= limit) + break; reader.deleteDocument(docID); any = true; } } } finally { - docs.close(); + //docs.close(); } // Delete by docID Index: src/java/org/apache/lucene/index/FieldsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FieldsConsumer.java (revision 0) +++ src/java/org/apache/lucene/index/FieldsConsumer.java (revision 0) @@ -0,0 +1,36 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** Abstract API that consumes terms, doc, freq, prox and + * payloads postings. Concrete implementations of this + * actually do "something" with the postings (write it into + * the index in a specific format). + * + * NOTE: this API is experimental and will likely change + */ +abstract class FieldsConsumer { + + /** Add a new field */ + abstract TermsConsumer addField(FieldInfo field) throws IOException; + + /** Called when we are done adding everything. */ + abstract void close() throws IOException; +} Index: src/java/org/apache/lucene/index/FieldsEnum.java =================================================================== --- src/java/org/apache/lucene/index/FieldsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/FieldsEnum.java (revision 0) @@ -0,0 +1,50 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.AttributeSource; + +/** Enumerates indexed fields. + * + * NOTE: this API is experimental and will likely change */ + +public abstract class FieldsEnum extends AttributeSource { + + /** Seeks to the specified field. Returns true if the field + * exists. */ + public abstract boolean seek(String field) throws IOException; + + // nocommit -- why not return String/null? + /** Increments the enumeration to the next field. True if one exists.*/ + public abstract boolean next() throws IOException; + + /** Returns the current Field.*/ + public abstract String field(); + + /** Get TermsEnum for the current field. You should not + * call {@link #next()} or {@link #seek()} until you're + * done using this TermsEnum. */ + public abstract TermsEnum terms() throws IOException; + + // nocommit -- maybe no close method? + /** Closes the enumeration to further activity, freeing resources. */ + //public abstract void close() throws IOException; +} + Index: src/java/org/apache/lucene/index/FieldsProducer.java =================================================================== --- src/java/org/apache/lucene/index/FieldsProducer.java (revision 0) +++ src/java/org/apache/lucene/index/FieldsProducer.java (revision 0) @@ -0,0 +1,34 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.util.BitVector; + +/** Abstract API that provides terms, doc, freq, prox and + * payloads postings. Concrete implementations of this + * actually do "something" to read the postings from some + * store. + * + * NOTE: this API is experimental and will likely change + */ + +abstract class FieldsProducer { + abstract FieldsEnum fields(BitVector deletedDocs) throws IOException; + abstract void close() throws IOException; +} Index: src/java/org/apache/lucene/index/FormatPostingsDocsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsDocsConsumer.java (revision 803321) +++ src/java/org/apache/lucene/index/FormatPostingsDocsConsumer.java (working copy) @@ -1,34 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -/** - * NOTE: this API is experimental and will likely change - */ - -abstract class FormatPostingsDocsConsumer { - - /** Adds a new doc in this term. If this returns null - * then we just skip consuming positions/payloads. */ - abstract FormatPostingsPositionsConsumer addDoc(int docID, int termDocFreq) throws IOException; - - /** Called when we are done adding docs to this term */ - abstract void finish() throws IOException; -} Index: src/java/org/apache/lucene/index/FormatPostingsDocsReader.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsDocsReader.java (revision 0) +++ src/java/org/apache/lucene/index/FormatPostingsDocsReader.java (revision 0) @@ -0,0 +1,427 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitVector; + +/** Concrete class that reads the current doc/freq/skip + * postings format */ + +// nocommit -- should we switch "hasProx" higher up? and +// create two separate docs readers, one that also reads +// prox and one that doesn't? + +class FormatPostingsDocsReader extends FormatPostingsTermsDictDocsReader { + + final IndexInput freqIn; + IndexInput termsIn; + + private final FormatPostingsPositionsReader posReader; + + int skipInterval; + int maxSkipLevels; + + FormatPostingsDocsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize) throws IOException { + freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.FREQ_EXTENSION), readBufferSize); + + boolean success = false; + try { + if (segmentInfo.getHasProx()) + posReader = new FormatPostingsPositionsReader(dir, segmentInfo, readBufferSize); + else + posReader = null; + success = true; + } finally { + if (!success) + freqIn.close(); + } + } + + static void files(SegmentInfo segmentInfo, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.FREQ_EXTENSION)); + FormatPostingsPositionsReader.files(segmentInfo, files); + } + + void start(IndexInput termsIn) throws IOException { + this.termsIn = termsIn; + + // Make sure we are talking to the matching past writer + PostingsCodec.checkHeader(termsIn, FormatPostingsDocsWriter.CODEC, FormatPostingsDocsWriter.VERSION_START); + + skipInterval = termsIn.readInt(); + maxSkipLevels = termsIn.readInt(); + if (posReader != null) + posReader.start(termsIn); + } + + Reader reader(FieldInfo fieldInfo, IndexInput termsIn) { + + final FormatPostingsPositionsReader.TermsDictReader posReader2; + if (posReader != null && !fieldInfo.omitTermFreqAndPositions) + posReader2 = (FormatPostingsPositionsReader.TermsDictReader) posReader.reader(fieldInfo, termsIn); + else + posReader2 = null; + + return new TermsDictReader(fieldInfo, posReader2, termsIn); + } + + void close() throws IOException { + try { + freqIn.close(); + } finally { + if (posReader != null) + posReader.close(); + } + } + + class TermsDictReader extends Reader { + + final IndexInput termsIn; + final FieldInfo fieldInfo; + long freqOffset; + long skipOffset; + int docFreq; + + // TODO: abstraction violation (we are storing this with + // the concrete impl. as the type, not the abstract base + // class) + final FormatPostingsPositionsReader.TermsDictReader posReader; + private SegmentDocsEnum docs; + + TermsDictReader(FieldInfo fieldInfo, FormatPostingsPositionsReader.TermsDictReader posReader, IndexInput termsIn) { + this.termsIn = termsIn; // not cloned + this.fieldInfo = fieldInfo; + this.posReader = posReader; + } + + void readTerm(int docFreq, boolean isIndexTerm) throws IOException { + + this.docFreq = docFreq; + if (PostingsCodec.DEBUG) + System.out.println(" dr.readTerm termsInPointer=" + termsIn.getFilePointer() + " df=" + docFreq + " isIndex=" + isIndexTerm); + + if (isIndexTerm) + freqOffset = termsIn.readVLong(); + else + freqOffset += termsIn.readVLong(); + + if (PostingsCodec.DEBUG) + System.out.println(" freqOffset=" + freqOffset + " vs len=" + freqIn.length()); + + if (docFreq >= skipInterval) + skipOffset = termsIn.readVLong(); + else + skipOffset = 0; + + if (posReader != null) + posReader.readTerm(docFreq, isIndexTerm); + } + + public void close() throws IOException { + if (posReader != null) + posReader.close(); + } + + DocsEnum docs(BitVector deletedDocs) throws IOException { + + if (docs == null) + // Lazy init + docs = new SegmentDocsEnum(); + + docs.init(deletedDocs); + + return docs; + } + + class SegmentDocsEnum extends DocsEnum { + int docFreq; + int doc; + int count; + int freq; + long skipStart; + long freqStart; + final IndexInput freqIn; + // nocommit -- should we do omitTF with 2 different enum classes? + final boolean omitTF; + private BitVector deletedDocs; + + // nocommit -- should we do hasProx with 2 different enum classes? + + boolean skipped; + DefaultSkipListReader skipper; + + // TODO: abstraction violation: we are storing the + // concrete impl, not the abstract base class + FormatPostingsPositionsReader.TermsDictReader.SegmentPositionsEnum positions; + + SegmentDocsEnum() { + if (PostingsCodec.DEBUG) + System.out.println("new docs enum"); + this.freqIn = (IndexInput) FormatPostingsDocsReader.this.freqIn.clone(); + omitTF = fieldInfo.omitTermFreqAndPositions; + if (omitTF) + freq = 1; + } + + public void close() { + } + + void init(BitVector deletedDocs) throws IOException { + if (PostingsCodec.DEBUG) + System.out.println("[" + desc + "] dr.init freqIn seek " + freqOffset + " this=" + this + " (in=" + freqIn + "; this=" + this + ")"); + this.deletedDocs = deletedDocs; + freqIn.seek(freqOffset); + this.docFreq = TermsDictReader.this.docFreq; + count = 0; + doc = 0; + skipped = false; + skipStart = freqStart + skipOffset; + proxSkipFreq = 0; + + // maybe not necessary? + proxSkipPayloadLength = -1; + + // nocommit: abstraction violation + if (posReader != null) { + proxOffset = posReader.proxOffset; + } + + if (positions != null) + positions.payloadLength = -1; + } + + public int next() throws IOException { + if (PostingsCodec.DEBUG) + System.out.println("dr [" + desc + "] next count=" + count + " vs df=" + docFreq + " freq pointer=" + freqIn.getFilePointer() + " (in=" + freqIn + "; this=" + this + ") + has del docs=" + (deletedDocs != null) ); + + // new Throwable().printStackTrace(System.out); + + while(true) { + if (count == docFreq) + return -1; + + count++; + + // Decode next doc/freq pair + final int code = freqIn.readVInt(); + if (PostingsCodec.DEBUG) + System.out.println(" read code=" + code); + if (omitTF) + doc += code; + else { + doc += code >>> 1; // shift off low bit + if ((code & 1) != 0) // if low bit is set + freq = 1; // freq is one + else + freq = freqIn.readVInt(); // else read freq + + if (positions != null) + positions.skip(freq); + else + proxSkipFreq += freq; + } + + if (deletedDocs == null || !deletedDocs.get(doc)) + break; + else if (PostingsCodec.DEBUG) + System.out.println(" doc=" + doc + " is deleted"); + } + + // nocommit + if (PostingsCodec.DEBUG && positions != null) + positions.desc = desc + ":" + doc; + + if (PostingsCodec.DEBUG) + System.out.println(" result doc=" + doc); + return doc; + } + + public int read(int[] docs, int[] freqs) throws IOException { + int i = 0; + final int length = docs.length; + while (i < length && count < docFreq) { + count++; + // manually inlined call to next() for speed + final int code = freqIn.readVInt(); + if (omitTF) { + doc += code; + freq = 1; + } else { + doc += code >>> 1; // shift off low bit + if ((code & 1) != 0) // if low bit is set + freq = 1; // freq is one + else + freq = freqIn.readVInt(); // else read freq + + if (positions != null) + positions.skip(freq); + else + proxSkipFreq += freq; + } + + if (deletedDocs == null || !deletedDocs.get(doc)) { + docs[i] = doc; + freqs[i] = freq; + ++i; + } + } + + return i; + } + + public int doc() { + return doc; + } + + public int ord() { + assert count > 0; + return count-1; + } + + public int freq() { + return freq; + } + + long proxOffset; + int proxSkipPayloadLength = -1; + int proxSkipFreq; + PositionsEnum fakePositions; + + public PositionsEnum positions() throws IOException { + if (positions == null) { + // Lazy init + if (posReader == null) { + // TermFreq was omitted from this field during + // indexing, which means we pretend termFreq is + // always 1 with that 1 occurrence having + // position 0 + if (fakePositions == null) + fakePositions = new FormatPostingsFakePositionsEnum(); + return fakePositions; + } else { + // TODO: abstraction violation + positions = (FormatPostingsPositionsReader.TermsDictReader.SegmentPositionsEnum) posReader.positions(); + if (PostingsCodec.DEBUG) + System.out.println("pos skip proxOffset=" + proxOffset + " payloadlen=" + proxSkipPayloadLength + " skipPosCount= " + proxSkipFreq); + positions.skip(proxOffset, proxSkipPayloadLength, proxSkipFreq); + } + } + + if (PostingsCodec.DEBUG) + positions.desc = desc + ":" + doc; + + positions.catchUp(freq); + + return positions; + } + + public int skipTo(int target) throws IOException { + + // TODO: jump right to next() if target is < X away + // from where we are now? + + if (PostingsCodec.DEBUG) + System.out.println("dr [" + desc + "]: skip to target=" + target); + + if (skipOffset > 0) { + + // There are enough docs in the posting to have + // skip data + if (skipper == null) + // Lazy init + skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); + + if (!skipped) { + + // We haven't already skipped for this posting, + // so now we init the skipper + + // TODO: this is abstraction violation; instead, + // skipper should interact with this as a + // private consumer + skipper.init(freqOffset+skipStart, + freqOffset, proxOffset, + docFreq, fieldInfo.storePayloads); + + if (PostingsCodec.DEBUG) + System.out.println(" skip reader base freqFP=" + (freqOffset+skipStart) + " freqFP=" + freqOffset + " proxFP=" + proxOffset); + + skipped = true; + } + + final int newCount = skipper.skipTo(target); + + if (newCount > count) { + + if (PostingsCodec.DEBUG) + System.out.println("dr [" + desc + "]: skipper moved to newCount=" + newCount + " freqFP=" + skipper.getFreqPointer() + " proxFP=" + skipper.getProxPointer() + " doc=" + skipper.getDoc()); + + // Skipper did move + freqIn.seek(skipper.getFreqPointer()); + count = newCount; + doc = skipper.getDoc(); + + // TODO: abstraction violation; this should be a + // private interaction b/w skipper & posReader + if (positions != null) + // nocommit -- should that be count? + positions.skip(skipper.getProxPointer(), skipper.getPayloadLength(), 0); + else { + proxOffset = skipper.getProxPointer(); + proxSkipPayloadLength = skipper.getPayloadLength(); + // nocommit -- should that be count? + proxSkipFreq = 0; + } + } else if (PostingsCodec.DEBUG) + System.out.println(" no skipping to be done"); + } else if (PostingsCodec.DEBUG) + System.out.println(" no skip data (#docs is too low)"); + + // Now, linear scan for the rest: + do { + if (next() == -1) + return -1; + } while (target > doc); + + return doc; + } + } + } +} + +/** Returned when someone asks for positions() enum on field + * with omitTf true */ +class FormatPostingsFakePositionsEnum extends PositionsEnum { + int next() { + return 0; + } + int getPayloadLength() { + return 0; + } + boolean hasPayload() { + return false; + } + byte[] getPayload(byte[] data, int offset) { + return null; + } +} Index: src/java/org/apache/lucene/index/FormatPostingsDocsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsDocsWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/FormatPostingsDocsWriter.java (working copy) @@ -22,39 +22,70 @@ import java.io.IOException; -import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.store.IndexOutput; -final class FormatPostingsDocsWriter extends FormatPostingsDocsConsumer { +final class FormatPostingsDocsWriter extends DocsConsumer { + final static String CODEC = "SingleFileDocFreqSkip"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; final IndexOutput out; - final FormatPostingsTermsWriter parent; final FormatPostingsPositionsWriter posWriter; final DefaultSkipListWriter skipListWriter; final int skipInterval; + final int maxSkipLevels; final int totalNumDocs; + IndexOutput termsOut; boolean omitTermFreqAndPositions; boolean storePayloads; + // Starts a new term + long lastFreqStart; long freqStart; FieldInfo fieldInfo; - FormatPostingsDocsWriter(SegmentWriteState state, FormatPostingsTermsWriter parent) throws IOException { + FormatPostingsDocsWriter(SegmentWriteState state) throws IOException { super(); - this.parent = parent; - final String fileName = IndexFileNames.segmentFileName(parent.parent.segment, IndexFileNames.FREQ_EXTENSION); + final String fileName = IndexFileNames.segmentFileName(state.segmentName, IndexFileNames.FREQ_EXTENSION); state.flushedFiles.add(fileName); - out = parent.parent.dir.createOutput(fileName); - totalNumDocs = parent.parent.totalNumDocs; + out = state.directory.createOutput(fileName); + totalNumDocs = state.numDocs; - // TODO: abstraction violation - skipInterval = parent.parent.termsOut.skipInterval; - skipListWriter = parent.parent.skipListWriter; - skipListWriter.setFreqOutput(out); + // nocommit -- abstraction violation + skipListWriter = new DefaultSkipListWriter(state.skipInterval, + state.maxSkipLevels, + state.numDocs, + out, + null); + + skipInterval = state.skipInterval; + maxSkipLevels = state.maxSkipLevels; posWriter = new FormatPostingsPositionsWriter(state, this); } + void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + PostingsCodec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + termsOut.writeInt(skipInterval); // write skipInterval + termsOut.writeInt(maxSkipLevels); // write maxSkipLevels + posWriter.start(termsOut); + } + + void startTerm() { + freqStart = out.getFilePointer(); + if (!omitTermFreqAndPositions) + posWriter.startTerm(); + skipListWriter.resetSkip(); + } + + // nocommit -- should we NOT reuse across fields? would + // be cleaner + + // Currently, this instance is re-used across fields, so + // our parent calls setField whenever the field changes void setField(FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions; @@ -64,13 +95,19 @@ int lastDocID; int df; + + int count; /** Adds a new doc in this term. If this returns null * then we just skip consuming positions/payloads. */ - FormatPostingsPositionsConsumer addDoc(int docID, int termDocFreq) throws IOException { + PositionsConsumer addDoc(int docID, int termDocFreq) throws IOException { final int delta = docID - lastDocID; + + if (PostingsCodec.DEBUG) + System.out.println(" dw.addDoc [" + desc + "] count=" + (count++) + " docID=" + docID + " lastDocID=" + lastDocID + " delta=" + delta + " omitTF=" + omitTermFreqAndPositions + " freq=" + termDocFreq + " freqPointer=" + out.getFilePointer()); + if (docID < 0 || (df > 0 && delta <= 0)) throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); @@ -78,8 +115,12 @@ // TODO: abstraction violation skipListWriter.setSkipData(lastDocID, storePayloads, posWriter.lastPayloadLength); skipListWriter.bufferSkip(df); + if (PostingsCodec.DEBUG) + System.out.println(" bufferSkip lastDocID=" + lastDocID + " df=" + df + " freqFP=" + out.getFilePointer() + " proxFP=" + skipListWriter.proxOutput.getFilePointer()); } + // nocommit -- move this assert up above; every consumer + // shouldn't have to check for this bug: assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; lastDocID = docID; @@ -92,36 +133,56 @@ out.writeVInt(termDocFreq); } - return posWriter; + // nocommit + if (PostingsCodec.DEBUG) + ((FormatPostingsPositionsWriter) posWriter).desc = desc + ":" + docID; + + if (omitTermFreqAndPositions) + return null; + else + return posWriter; } - private final TermInfo termInfo = new TermInfo(); // minimize consing - final UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result(); - /** Called when we are done adding docs to this term */ - void finish() throws IOException { - long skipPointer = skipListWriter.writeSkip(out); + void finishTerm(int docCount, boolean isIndexTerm) throws IOException { + // nocommit -- wasteful we are counting this in two places? + assert docCount == df; + if (PostingsCodec.DEBUG) + System.out.println("dw.finishTerm termsOut pointer=" + termsOut.getFilePointer() + " freqStart=" + freqStart + " df=" + df); - // TODO: this is abstraction violation -- we should not - // peek up into parents terms encoding format - termInfo.set(df, parent.freqStart, parent.proxStart, (int) (skipPointer - parent.freqStart)); + if (isIndexTerm) + // Write absolute at seek points + termsOut.writeVLong(freqStart); + else + // Write delta between seek points + termsOut.writeVLong(freqStart - lastFreqStart); - // TODO: we could do this incrementally - UnicodeUtil.UTF16toUTF8(parent.currentTerm, parent.currentTermStart, utf8); + lastFreqStart = freqStart; - if (df > 0) { - parent.termsOut.add(fieldInfo.number, - utf8.result, - utf8.length, - termInfo); - } + if (df >= skipInterval) { + if (PostingsCodec.DEBUG) + System.out.println(" writeSkip @ freqFP=" + out.getFilePointer() + " freqStartFP=" + freqStart); + termsOut.writeVLong(skipListWriter.writeSkip(out)-freqStart); + } + + if (!omitTermFreqAndPositions) + posWriter.finishTerm(isIndexTerm); + lastDocID = 0; df = 0; + + // nocommit + count = 0; } void close() throws IOException { - out.close(); - posWriter.close(); + if (PostingsCodec.DEBUG) + System.out.println("docs writer close pointer=" + out.getFilePointer()); + try { + out.close(); + } finally { + posWriter.close(); + } } } Index: src/java/org/apache/lucene/index/FormatPostingsFieldsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsFieldsConsumer.java (revision 803321) +++ src/java/org/apache/lucene/index/FormatPostingsFieldsConsumer.java (working copy) @@ -1,36 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -/** Abstract API that consumes terms, doc, freq, prox and - * payloads postings. Concrete implementations of this - * actually do "something" with the postings (write it into - * the index in a specific format). - * - * NOTE: this API is experimental and will likely change - */ -abstract class FormatPostingsFieldsConsumer { - - /** Add a new field */ - abstract FormatPostingsTermsConsumer addField(FieldInfo field) throws IOException; - - /** Called when we are done adding everything. */ - abstract void finish() throws IOException; -} Index: src/java/org/apache/lucene/index/FormatPostingsFieldsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsFieldsWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/FormatPostingsFieldsWriter.java (working copy) @@ -1,73 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.Directory; - -final class FormatPostingsFieldsWriter extends FormatPostingsFieldsConsumer { - - final Directory dir; - final String segment; - final TermInfosWriter termsOut; - final FieldInfos fieldInfos; - final FormatPostingsTermsWriter termsWriter; - final DefaultSkipListWriter skipListWriter; - final int totalNumDocs; - - public FormatPostingsFieldsWriter(SegmentWriteState state, FieldInfos fieldInfos) throws IOException { - super(); - - dir = state.directory; - segment = state.segmentName; - totalNumDocs = state.numDocs; - this.fieldInfos = fieldInfos; - termsOut = new TermInfosWriter(dir, - segment, - fieldInfos, - state.termIndexInterval); - - // TODO: this is a nasty abstraction violation (that we - // peek down to find freqOut/proxOut) -- we need a - // better abstraction here whereby these child consumers - // can provide skip data or not - skipListWriter = new DefaultSkipListWriter(termsOut.skipInterval, - termsOut.maxSkipLevels, - totalNumDocs, - null, - null); - - state.flushedFiles.add(state.segmentFileName(IndexFileNames.TERMS_EXTENSION)); - state.flushedFiles.add(state.segmentFileName(IndexFileNames.TERMS_INDEX_EXTENSION)); - - termsWriter = new FormatPostingsTermsWriter(state, this); - } - - /** Add a new field */ - FormatPostingsTermsConsumer addField(FieldInfo field) { - termsWriter.setField(field); - return termsWriter; - } - - /** Called when we are done adding everything. */ - void finish() throws IOException { - termsOut.close(); - termsWriter.close(); - } -} Index: src/java/org/apache/lucene/index/FormatPostingsPositionsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsPositionsConsumer.java (revision 803321) +++ src/java/org/apache/lucene/index/FormatPostingsPositionsConsumer.java (working copy) @@ -1,32 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.IndexInput; - -abstract class FormatPostingsPositionsConsumer { - - /** Add a new position & payload. If payloadLength > 0 - * you must read those bytes from the IndexInput. */ - abstract void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException; - - /** Called when we are done adding positions & payloads */ - abstract void finish() throws IOException; -} Index: src/java/org/apache/lucene/index/FormatPostingsPositionsReader.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsPositionsReader.java (revision 0) +++ src/java/org/apache/lucene/index/FormatPostingsPositionsReader.java (revision 0) @@ -0,0 +1,234 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.util.BitVector; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.Directory; + +// nocommit -- base class should not be named terms dict: +// this class interacts w/ a docsreader +class FormatPostingsPositionsReader extends FormatPostingsTermsDictPositionsReader { + + final IndexInput proxIn; + IndexInput termsIn; + + FormatPostingsPositionsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize) throws IOException { + assert segmentInfo.getHasProx(); + proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.PROX_EXTENSION), readBufferSize); + } + + void start(IndexInput termsIn) throws IOException { + this.termsIn = termsIn; + + PostingsCodec.checkHeader(termsIn, FormatPostingsPositionsWriter.CODEC, FormatPostingsPositionsWriter.VERSION_START); + } + + static void files(SegmentInfo segmentInfo, Collection files) { + if (segmentInfo.getHasProx()) + files.add(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.PROX_EXTENSION)); + } + + Reader reader(FieldInfo fieldInfo, IndexInput termsIn) { + return new TermsDictReader(termsIn, fieldInfo); + } + + void close() throws IOException { + if (proxIn != null) + proxIn.close(); + } + + class TermsDictReader extends Reader { + + final IndexInput termsIn; + final FieldInfo fieldInfo; + long proxOffset; + + TermsDictReader(IndexInput termsIn, FieldInfo fieldInfo) { + this.termsIn = termsIn; + this.fieldInfo = fieldInfo; + } + + void readTerm(int docFreq, boolean isIndexTerm) throws IOException { + if (PostingsCodec.DEBUG) + System.out.println(" pr.readterm termsInPointer=" + termsIn.getFilePointer() + " isIndex=" + isIndexTerm); + if (isIndexTerm) + proxOffset = termsIn.readVLong(); + else + proxOffset += termsIn.readVLong(); + if (PostingsCodec.DEBUG) + System.out.println(" proxOffset=" + proxOffset); + if (positions != null) { + positions.seekPending = true; + positions.skipOffset = proxOffset; + positions.skipPosCount = 0; + } + } + + void close() throws IOException { + } + + SegmentPositionsEnum positions; + + PositionsEnum positions() throws IOException { + + if (positions == null) + // Lazy init + positions = new SegmentPositionsEnum(); + + return positions; + } + + // nocommit -- should we have different reader for + // payload vs no payload? + class SegmentPositionsEnum extends PositionsEnum { + + // nocommit + String desc; + + final IndexInput proxIn; + + final boolean storePayloads; + + boolean seekPending; // True if we must seek before reading next position + boolean payloadPending; // True if we must skip payload beore reading next position + + long skipOffset; + int skipPosCount; + + int position; + int payloadLength; + + SegmentPositionsEnum() { + if (PostingsCodec.DEBUG) + System.out.println("new pos enum"); + proxIn = (IndexInput) FormatPostingsPositionsReader.this.proxIn.clone(); + storePayloads = fieldInfo.storePayloads; + } + + void skip(long proxOffset, int lastPayloadLength, int numPositions) { + skipOffset = proxOffset; + payloadLength = lastPayloadLength; + assert payloadLength >= 0 || payloadLength == -1; + skipPosCount = numPositions; + seekPending = true; + payloadPending = false; + if (PostingsCodec.DEBUG) + System.out.println("pr [" + desc + "] skip fp= " + proxOffset + " numPositions=" + numPositions); + } + + void skip(int numPositions) { + skipPosCount += numPositions; + if (PostingsCodec.DEBUG) + System.out.println("pr [" + desc + "] skip " + numPositions + " positions; now " + skipPosCount); + } + + void catchUp(int currentCount) throws IOException { + if (PostingsCodec.DEBUG) + System.out.println(" pos catchup: seekPending=" + seekPending + " skipOffset=" + skipOffset + " skipPosCount " + skipPosCount + " vs currentCount " + currentCount + " payloadLen=" + payloadLength); + if (seekPending) { + proxIn.seek(skipOffset); + seekPending = false; + } + + while(skipPosCount > currentCount) + next(); + if (PostingsCodec.DEBUG) + System.out.println(" pos catchup done"); + positions.init(); + } + + void init() { + if (PostingsCodec.DEBUG) + System.out.println(" pos init"); + position = 0; + } + + int next() throws IOException { + + if (PostingsCodec.DEBUG) + System.out.println(" pr.next [" + desc + "]: fp=" + proxIn.getFilePointer() + " return pos=" + position); + + if (storePayloads) { + + if (payloadPending && payloadLength > 0) { + if (PostingsCodec.DEBUG) + System.out.println(" payload pending: skip " + payloadLength + " bytes"); + proxIn.seek(proxIn.getFilePointer()+payloadLength); + } + + final int code = proxIn.readVInt(); + if ((code & 1) != 0) { + // Payload length has changed + payloadLength = proxIn.readVInt(); + assert payloadLength >= 0; + if (PostingsCodec.DEBUG) + System.out.println(" new payloadLen=" + payloadLength); + } + assert payloadLength != -1; + + payloadPending = true; + position += code >>> 1; + } else + position += proxIn.readVInt(); + + skipPosCount--; + + // NOTE: the old API actually allowed this... + assert skipPosCount >= 0: "next() was called too many times (more than FormatPostingsDocsEnum.freq() times)"; + + if (PostingsCodec.DEBUG) + System.out.println(" proxFP=" + proxIn.getFilePointer() + " return pos=" + position); + return position; + } + + int getPayloadLength() { + return payloadLength; + } + + byte[] getPayload(byte[] data, int offset) throws IOException { + + if (!payloadPending) + throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); + + final byte[] retArray; + final int retOffset; + if (data == null || data.length-offset < payloadLength) { + // the array is too small to store the payload data, + // so we allocate a new one + retArray = new byte[payloadLength]; + retOffset = 0; + } else { + retArray = data; + retOffset = offset; + } + + proxIn.readBytes(retArray, retOffset, payloadLength); + payloadPending = false; + return retArray; + } + + public boolean hasPayload() { + return payloadPending && payloadLength > 0; + } + } + } +} Index: src/java/org/apache/lucene/index/FormatPostingsPositionsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsPositionsWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/FormatPostingsPositionsWriter.java (working copy) @@ -22,30 +22,54 @@ import java.io.IOException; -final class FormatPostingsPositionsWriter extends FormatPostingsPositionsConsumer { +final class FormatPostingsPositionsWriter extends PositionsConsumer { + final static String CODEC = "SingleFilePositionsPayloads"; + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + final FormatPostingsDocsWriter parent; final IndexOutput out; + + IndexOutput termsOut; boolean omitTermFreqAndPositions; boolean storePayloads; int lastPayloadLength = -1; + // nocommit + String desc; + FormatPostingsPositionsWriter(SegmentWriteState state, FormatPostingsDocsWriter parent) throws IOException { this.parent = parent; omitTermFreqAndPositions = parent.omitTermFreqAndPositions; - if (parent.parent.parent.fieldInfos.hasProx()) { + if (state.fieldInfos.hasProx()) { // At least one field does not omit TF, so create the // prox file - final String fileName = IndexFileNames.segmentFileName(parent.parent.parent.segment, IndexFileNames.PROX_EXTENSION); + final String fileName = IndexFileNames.segmentFileName(state.segmentName, IndexFileNames.PROX_EXTENSION); state.flushedFiles.add(fileName); - out = parent.parent.parent.dir.createOutput(fileName); + out = state.directory.createOutput(fileName); parent.skipListWriter.setProxOutput(out); } else // Every field omits TF so we will write no prox file out = null; } + void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + PostingsCodec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + } + + long proxStart; + long lastProxStart; + + void startTerm() { + proxStart = out.getFilePointer(); + lastPayloadLength = -1; + } + + int lastPosition; /** Add a new position & payload */ @@ -53,11 +77,27 @@ assert !omitTermFreqAndPositions: "omitTermFreqAndPositions is true"; assert out != null; + if (PostingsCodec.DEBUG) + if (payload != null) + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " fp=" + out.getFilePointer() + " payload=" + payloadLength + " bytes"); + else + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " fp=" + out.getFilePointer()); + + final int delta = position - lastPosition; + + assert delta > 0 || position == 0 || position == -1: "position=" + position + " lastPosition=" + lastPosition; // not quite right (if pos=0 is repeated twice we don't catch it) + lastPosition = position; if (storePayloads) { + if (PostingsCodec.DEBUG) + System.out.println(" store payloads"); + if (payloadLength != lastPayloadLength) { + if (PostingsCodec.DEBUG) + System.out.println(" payload len change old=" + lastPayloadLength + " new=" + payloadLength); + lastPayloadLength = payloadLength; out.writeVInt((delta<<1)|1); out.writeVInt(payloadLength); @@ -75,11 +115,25 @@ } /** Called when we are done adding positions & payloads */ - void finish() { + void finishDoc() { lastPosition = 0; - lastPayloadLength = -1; } + void finishTerm(boolean isIndexTerm) throws IOException { + assert !omitTermFreqAndPositions; + + if (PostingsCodec.DEBUG) + System.out.println("poswriter finishTerm isIndex=" + isIndexTerm + " proxStart=" + proxStart + " pointer=" + termsOut.getFilePointer()); + + if (isIndexTerm) + // Write absolute at seek points + termsOut.writeVLong(proxStart); + else + termsOut.writeVLong(proxStart-lastProxStart); + + lastProxStart = proxStart; + } + void close() throws IOException { if (out != null) out.close(); Index: src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java (revision 803321) +++ src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java (working copy) @@ -1,46 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.ArrayUtil; - -/** - * NOTE: this API is experimental and will likely change - */ - -abstract class FormatPostingsTermsConsumer { - - /** Adds a new term in this field; term ends with U+FFFF - * char */ - abstract FormatPostingsDocsConsumer addTerm(char[] text, int start) throws IOException; - - char[] termBuffer; - FormatPostingsDocsConsumer addTerm(String text) throws IOException { - final int len = text.length(); - if (termBuffer == null || termBuffer.length < 1+len) - termBuffer = new char[ArrayUtil.getNextSize(1+len)]; - text.getChars(0, len, termBuffer, 0); - termBuffer[len] = 0xffff; - return addTerm(termBuffer, 0); - } - - /** Called when we are done adding terms to this field */ - abstract void finish() throws IOException; -} Index: src/java/org/apache/lucene/index/FormatPostingsTermsDictDocsReader.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsTermsDictDocsReader.java (revision 0) +++ src/java/org/apache/lucene/index/FormatPostingsTermsDictDocsReader.java (revision 0) @@ -0,0 +1,46 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitVector; + +/** TermsDictReader interacts with a single instance of this + * to manage creation of multiple docs enum + * instances. */ +abstract class FormatPostingsTermsDictDocsReader { + + abstract class Reader { + abstract void readTerm(int docFreq, boolean isIndexTerm) throws IOException; + + /** Returns a docs enum for the last term read */ + abstract DocsEnum docs(BitVector deletedDocs) throws IOException; + + abstract void close() throws IOException; + } + + abstract void start(IndexInput termsIn) throws IOException; + + /** Returns a new private reader for stepping through + * terms, getting DocsEnum. */ + abstract Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException; + + abstract void close() throws IOException; +} Index: src/java/org/apache/lucene/index/FormatPostingsTermsDictPositionsReader.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsTermsDictPositionsReader.java (revision 0) +++ src/java/org/apache/lucene/index/FormatPostingsTermsDictPositionsReader.java (revision 0) @@ -0,0 +1,42 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; + +// nocommit -- bad name: this class never interacts directly +// w/ termsdict +abstract class FormatPostingsTermsDictPositionsReader { + + abstract class Reader { + abstract void readTerm(int docFreq, boolean isIndexTerm) throws IOException; + + /** Returns a pos enum for the last term read */ + abstract PositionsEnum positions() throws IOException; + + abstract void close() throws IOException; + } + + abstract void start(IndexInput termsIn) throws IOException; + + abstract Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException; + + abstract void close() throws IOException; +} Index: src/java/org/apache/lucene/index/FormatPostingsTermsDictReader.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsTermsDictReader.java (revision 0) +++ src/java/org/apache/lucene/index/FormatPostingsTermsDictReader.java (revision 0) @@ -0,0 +1,514 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.Collection; +import java.util.Iterator; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BitVector; + +class FormatPostingsTermsDictReader extends FieldsProducer { + private final IndexInput in; + private final IndexInput indexIn; + private final int indexInterval; + private final FormatPostingsTermsDictDocsReader docs; + + private int indexDivisor = 1; + private boolean anyIndexRead; + + int totalIndexInterval; + + private final FieldInfos fieldInfos; + final TreeMap fields = new TreeMap(); + + private final String segment; + + FormatPostingsTermsDictReader(Directory dir, FieldInfos fieldInfos, String segment, FormatPostingsTermsDictDocsReader docs, int readBufferSize, int indexDivisor) throws IOException { + in = dir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.TERMS_EXTENSION), readBufferSize); + this.indexDivisor = indexDivisor; + boolean success = false; + try { + indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.TERMS_INDEX_EXTENSION), readBufferSize); + success = true; + } finally { + if (!success) + in.close(); + } + + success = false; + try { + + this.fieldInfos = fieldInfos; + this.segment = segment; + int format = in.readInt(); + if (format != FormatPostingsTermsDictWriter.FORMAT) + throw new CorruptIndexException("format mismatch"); + + final long dirOffset = in.readLong(); + indexInterval = in.readInt(); + totalIndexInterval = indexInterval; + + this.docs = docs; + docs.start(in); + in.seek(dirOffset); + + final int numFields = in.readInt(); + + if (PostingsCodec.DEBUG) + System.out.println("tdr create seg=" + segment + " numFields=" + numFields); + + // nocommit -- why did i want to order by field number? + for(int i=0;i 0) { + final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); + fields.put(fieldInfo.name, new FieldReader(fieldInfo, numTerms, termsStartPointer, indexStartPointer)); + } + } + success = true; + } finally { + if (!success) { + try { + in.close(); + } finally { + indexIn.close(); + } + } + } + } + + public void close() throws IOException { + try { + in.close(); + } finally { + try { + indexIn.close(); + } finally { + docs.close(); + } + } + } + + static void files(SegmentInfo segmentInfo, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.TERMS_INDEX_EXTENSION)); + } + + /** + *

Sets the indexDivisor, which subsamples the number + * of indexed terms loaded into memory. This has a + * similar effect as {@link + * IndexWriter#setTermIndexInterval} except that setting + * must be done at indexing time while this setting can be + * set per reader. When set to N, then one in every + * N*termIndexInterval terms in the index is loaded into + * memory. By setting this to a value > 1 you can reduce + * memory usage, at the expense of higher latency when + * loading a TermInfo. The default value is 1.

+ * + * NOTE: you must call this before the term + * index is loaded. If the index is already loaded, + * an IllegalStateException is thrown. + * + + @throws IllegalStateException if the term index has + * already been loaded into memory. + */ + public void setIndexDivisor(int indexDivisor) throws IllegalStateException { + if (indexDivisor < 1) + throw new IllegalArgumentException("indexDivisor must be > 0: got " + indexDivisor); + + if (anyIndexRead) + throw new IllegalStateException("index terms are already loaded"); + + this.indexDivisor = indexDivisor; + totalIndexInterval = indexInterval * indexDivisor; + } + + /** Returns the indexDivisor. + * @see #setIndexDivisor + */ + public int getIndexDivisor() { + return indexDivisor; + } + + public FieldsEnum fields(BitVector deletedDocs) { + if (PostingsCodec.DEBUG) + System.out.println("tdr.fields(): field count=" + fields.size()); + return new Fields(deletedDocs); + } + + private class Fields extends FieldsEnum { + Iterator it; + FieldReader current; + final BitVector deletedDocs; + + Fields(BitVector deletedDocs) { + this.deletedDocs = deletedDocs; + } + + public boolean next() { + if (PostingsCodec.DEBUG) { + System.out.println("tdrf.next seg=" + segment); + new Throwable().printStackTrace(System.out); + } + if (it == null) { + if (PostingsCodec.DEBUG) + System.out.println(" init it"); + it = fields.values().iterator(); + } + if (it.hasNext()) { + current = (FieldReader) it.next(); + if (PostingsCodec.DEBUG) + System.out.println(" hasNext set current field=" + current.fieldInfo.name); + return true; + } else + return false; + } + + public String field() { + return current.fieldInfo.name; + } + + public boolean seek(String field) { + if (PostingsCodec.DEBUG) + System.out.println("tdrf.seek field=" + field + " seg=" + segment); + + it = fields.tailMap(field).values().iterator(); + if (next()) { + if (PostingsCodec.DEBUG) + System.out.println(" return " + field.equals(current.fieldInfo.name)); + return field.equals(current.fieldInfo.name); + } else + return false; + } + + public TermsEnum terms() throws IOException { + return current.terms(deletedDocs); + } + + public void close() {} + } + + private class FieldReader { + + final long numTerms; + final FieldInfo fieldInfo; + final long indexStartPointer; + final long termsStartPointer; + + // TODO: genericize "skipper" API so that we could swap + // in a multi-level skipper, here, instead of flat one: + // TODO: we could save mem here by packing our own shared char[]'s + String[] indexTerms; + long[] indexOffsets; + + FieldReader(FieldInfo fieldInfo, long numTerms, long termsStartPointer, long indexStartPointer) { + this.fieldInfo = fieldInfo; + this.numTerms = numTerms; + assert numTerms > 0; + this.indexStartPointer = indexStartPointer; + this.termsStartPointer = termsStartPointer; + } + + synchronized final void readIndex() throws IOException { + if (indexTerms != null) + return; + + final int indexSize = (int) (1+(numTerms-1)/totalIndexInterval); + + if (PostingsCodec.DEBUG) + System.out.println(" tdr.readIndex field=" + fieldInfo.name + " numTerms=" + numTerms + " indexSize=" + indexSize + " indexSeek=" + indexStartPointer + " segment=" + segment + " indexDivisor=" + indexDivisor); + + IndexInput in = (IndexInput) indexIn.clone(); + in.seek(indexStartPointer); + + indexTerms = new String[indexSize]; + indexOffsets = new long[indexSize]; + + if (PostingsCodec.DEBUG) + System.out.println("read index for field=" + fieldInfo.name); + + long pointer = termsStartPointer; + final DeltaBytesReader bytesReader = new DeltaBytesReader(in); + final int numIndexTerms = (int) (1+(numTerms-1)/indexInterval); + int upto = 0; + for(int i=0;i= numTerms) { + termUpto++; + return false; + } + if (PostingsCodec.DEBUG) { + System.out.println("tdr.next: field=" + fieldInfo.name + " termsInPointer=" + in.getFilePointer() + " vs len=" + in.length() + " isIndex=" + (termUpto%indexInterval==0) + " seg=" + segment); + //new Throwable().printStackTrace(System.out); + } + bytesReader.read(); + docFreq = in.readVInt(); + if (PostingsCodec.DEBUG) + System.out.println(" text=" + bytesReader.text() + " freq=" + docFreq); + docs.readTerm(docFreq, termUpto % indexInterval == 0); + termUpto++; + if (PostingsCodec.DEBUG) + System.out.println(" termUpto=" + termUpto + " vs numTerms=" + numTerms + " fp=" + in.getFilePointer()); + return true; + } + + public int docFreq() { + if (termUpto >= 1+numTerms) + return 0; + else + return docFreq; + } + + public String text() { + // nocommit -- really necessary? + if (termUpto >= 1+numTerms) + return null; + else + return bytesReader.text(); + } + + public long ord() { + return termUpto-1; + } + + public DocsEnum docs() throws IOException { + doSkip = false; + // nocommit + DocsEnum docsEnum = docs.docs(deletedDocs); + docsEnum.desc = fieldInfo.name + ":" + bytesReader.text(); + return docsEnum; + } + + public void close() throws IOException { + in.close(); + docs.close(); + } + } + } + + private static class DeltaBytesReader { + private byte[] bytes; + final UnicodeUtil.UTF16Result chars = new UnicodeUtil.UTF16Result(); + final UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result(); + private int length; + final IndexInput in; + boolean started; + + DeltaBytesReader(IndexInput in) { + this.in = in; + bytes = new byte[10]; + } + + void reset(String text) { + UnicodeUtil.UTF16toUTF8(text, 0, text.length(), utf8); + if (utf8.length > bytes.length) + bytes = ArrayUtil.grow(bytes, utf8.length); + System.arraycopy(utf8.result, 0, + this.bytes, 0, utf8.length); + this.length = utf8.length; + chars.copyText(text); + } + + String text() { + // nocommit -- cache this? + return new String(chars.result, 0, chars.length); + } + + int compareTo(String other) { + + final int otherLength = other.length(); + final int minLength; + if (otherLength < chars.length) + minLength = otherLength; + else + minLength = chars.length; + + for(int i=0;i otherC) + return 1; + } + + if (chars.length < otherLength) + return -1; + else if (chars.length > otherLength) + return 1; + else + return 0; + } + + void read() throws IOException { + //System.out.println("terms reader fp=" + in.getFilePointer() + " this=" + this); + final int start = in.readVInt(); + final int suffix = in.readVInt(); + //System.out.println(" start=" + start + " suffix=" + suffix); + assert start <= length: "start=" + start + " length=" + length; + + if (start + suffix > bytes.length) + bytes = ArrayUtil.grow(bytes, start+suffix); + in.readBytes(bytes, start, suffix); + length = start + suffix; + + // TODO: conversion could be incremental + UnicodeUtil.UTF8toUTF16(bytes, 0, length, chars); + started = true; + } + } + +} \ No newline at end of file Index: src/java/org/apache/lucene/index/FormatPostingsTermsDictWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPostingsTermsDictWriter.java (revision 0) +++ src/java/org/apache/lucene/index/FormatPostingsTermsDictWriter.java (revision 0) @@ -0,0 +1,250 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.ArrayUtil; + +/** + * Writes terms dict and interacts with docs/positions + * consumers to write the postings files. + * + * The [new] terms dict format is field-centric: each field + * has its own section in the file. Fields are written in + * UTF16 string comparison order. Within each field, each + * term's text is written in UTF16 string comparison order. + */ + +class FormatPostingsTermsDictWriter extends FieldsConsumer { + + // Initial format + public static final int FORMAT = -1; + + public static final int FORMAT_CURRENT = FORMAT; + + private final int indexInterval; + private final DeltaBytesWriter termWriter; + private final DeltaBytesWriter termIndexWriter; + + final IndexOutput out; + final IndexOutput indexOut; + final DocsConsumer consumer; + final FieldInfos fieldInfos; + FieldInfo currentField; + + private List fields = new ArrayList(); + + // nocommit + private String segment; + + FormatPostingsTermsDictWriter(SegmentWriteState state, DocsConsumer consumer) throws IOException { + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, IndexFileNames.TERMS_EXTENSION); + out = state.directory.createOutput(termsFileName); + state.flushedFiles.add(termsFileName); + this.segment = state.segmentName; + + if (PostingsCodec.DEBUG) { + System.out.println("tdw: write to segment=" + state.segmentName); + new Throwable().printStackTrace(System.out); + } + + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, IndexFileNames.TERMS_INDEX_EXTENSION); + indexOut = state.directory.createOutput(indexFileName); + state.flushedFiles.add(indexFileName); + + fieldInfos = state.fieldInfos; + indexInterval = state.termIndexInterval; + + // Count indexed fields up front + final int numFields = fieldInfos.size(); + + out.writeInt(FORMAT_CURRENT); // write format + out.writeLong(0); // leave space for end index pointer + out.writeInt(indexInterval); // write indexInterval + + termWriter = new DeltaBytesWriter(out); + termIndexWriter = new DeltaBytesWriter(indexOut); + currentField = null; + this.consumer = consumer; + + consumer.start(out); // have consumer write its format/header + } + + TermsConsumer addField(FieldInfo field) { + if (PostingsCodec.DEBUG) + System.out.println("tdw.addField: field=" + field.name); + assert currentField == null || currentField.name.compareTo(field.name) < 0; + currentField = field; + TermsConsumer terms = new TermsWriter(field, consumer); + fields.add(terms); + return terms; + } + + void close() throws IOException { + if (PostingsCodec.DEBUG) + System.out.println("tdw.close seg=" + segment); + try { + final long indexPointer = out.getFilePointer(); + final int fieldCount = fields.size(); + + if (PostingsCodec.DEBUG) + System.out.println(" numFields=" + fieldCount); + + out.writeInt(fieldCount); + for(int i=0;i>>1; + if ((code & 1) != 0) + doc.numPositions = 1; + else + doc.numPositions = termsIn.readVInt(); + + if (doc.numPositions > doc.positions.length) + doc.reallocPositions(doc.numPositions); + + int position = 0; + int payloadLength = -1; + + for(int j=0;j>> 1; + if ((code2 & 1) != 0) + payloadLength = termsIn.readVInt(); + if (payloadLength > 0) { + if (pos.payload == null || payloadLength > pos.payload.length) + pos.payload = new byte[ArrayUtil.getNextSize(payloadLength)]; + termsIn.readBytes(pos.payload, 0, payloadLength); + } + } else + position += code2; + pos.pos = position; + pos.payloadLength = payloadLength; + } + } + doc.docID = docID; + } + + } else { + if (PostingsCodec.DEBUG) + System.out.println(" not pulsed pass isIndex=" + pendingIndexTerm); + + postingsReader.readTerm(docFreq, pendingIndexTerm); + pendingIndexTerm = false; + } + } + + public void close() throws IOException { + postingsReader.close(); + } + + final PulsingDocsEnum docsEnum = new PulsingDocsEnum(); + + DocsEnum docs(BitVector deletedDocs) throws IOException { + if (docFreq <= maxPulsingDocFreq) { + docsEnum.reset(deletedDocs); + return docsEnum; + } else + return postingsReader.docs(deletedDocs); + } + + class PulsingDocsEnum extends DocsEnum { + int nextRead; + private BitVector deletedDocs; + private Document doc; + + public void close() {} + + void reset(BitVector deletedDocs) { + this.deletedDocs = deletedDocs; + nextRead = 0; + } + + public int next() { + while(true) { + if (nextRead >= docFreq) + return -1; + else { + doc = docs[nextRead++]; + if (deletedDocs == null || !deletedDocs.get(doc.docID)) + return doc.docID; + } + } + } + + public int read(int[] retDocs, int[] retFreqs) { + final int limit; + int i=0; + // nocommit -- ob1? + while(nextRead < docFreq) { + doc = docs[nextRead++]; + if (deletedDocs == null || !deletedDocs.get(doc.docID)) { + retDocs[i] = doc.docID; + if (omitTF) + retFreqs[i] = 0; + else + retFreqs[i] = doc.numPositions; + i++; + } + } + return i; + } + + public int ord() { + assert nextRead <= docFreq; + return nextRead-1; + } + + public int freq() { + return doc.numPositions; + } + + class PulsingPositionsEnum extends PositionsEnum { + int nextRead; + FormatPulsingDocsWriter.Position pos; + + void reset() { + nextRead = 0; + } + + public int next() { + assert nextRead < doc.numPositions; + pos = doc.positions[nextRead++]; + return pos.pos; + } + + public int getPayloadLength() { + return pos.payloadLength; + } + + public boolean hasPayload() { + return pos.payloadLength > 0; + } + + public byte[] getPayload(byte[] data, int offset) { + // nocommit -- inefficient + System.arraycopy(pos.payload, 0, data, offset, pos.payloadLength); + return data; + } + } + + final PulsingPositionsEnum positions = new PulsingPositionsEnum(); + + public PositionsEnum positions() throws IOException { + positions.reset(); + return positions; + } + + public int skipTo(int target) throws IOException { + int doc; + while((doc=next()) != -1) { + if (doc >= target) + return doc; + } + return -1; + } + } + } +} Index: src/java/org/apache/lucene/index/FormatPulsingDocsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatPulsingDocsWriter.java (revision 0) +++ src/java/org/apache/lucene/index/FormatPulsingDocsWriter.java (revision 0) @@ -0,0 +1,274 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Consumes doc & freq, writing them using the current + * index file format */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ArrayUtil; + +final class FormatPulsingDocsWriter extends DocsConsumer { + + final static String CODEC = "PulsedPostings"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + IndexOutput termsOut; + + boolean omitTF; + boolean storePayloads; + + // Starts a new term + FieldInfo fieldInfo; + + // nocommit + String desc; + + static class Document { + int docID; + int termDocFreq; + int numPositions; + Position[] positions; + Document() { + positions = new Position[1]; + positions[0] = new Position(); + } + + void reallocPositions(int minSize) { + final Position[] newArray = new Position[ArrayUtil.getNextSize(minSize)]; + System.arraycopy(positions, 0, newArray, 0, positions.length); + for(int i=positions.length;i maxPulsingDocFreq docs + + static class Position { + byte[] payload; + int pos; + int payloadLength; + } + + // nocommit -- lazy init this? ie, if every single term + // was pulsed then we never need to use this fallback? + // Fallback writer for non-pulsed terms: + final DocsConsumer postingsDocsWriter; + + /** If docFreq <= maxPulsingDocFreq, its postings are + * inlined into terms dict */ + + FormatPulsingDocsWriter(SegmentWriteState state, int maxPulsingDocFreq, DocsConsumer postingsDocsWriter) throws IOException { + super(); + + pendingDocs = new Document[maxPulsingDocFreq]; + for(int i=0;i 0) { + if (pos.payload == null || payloadLength > pos.payload.length) + pos.payload = new byte[ArrayUtil.getNextSize(payloadLength)]; + System.arraycopy(payload, payloadOffset, pos.payload, 0, payloadLength); + pos.payloadLength = payloadLength; + } else + pos.payloadLength = 0; + } + void finishDoc() { + assert currentDoc.numPositions == currentDoc.termDocFreq; + } + void finishTerm(boolean isIndexTerm) {} + void close() {} + } + + final PositionsWriter posWriter = new PositionsWriter(); + + PositionsConsumer addDoc(int docID, int termDocFreq) throws IOException { + + assert docID >= 0: "got docID=" + docID; + + if (PostingsCodec.DEBUG) + System.out.println("PW.addDoc: docID=" + docID + " pendingDocCount=" + pendingDocCount + " vs " + pendingDocs.length + " pulsed=" + pulsed); + + if (!pulsed && pendingDocCount == pendingDocs.length) { + + // OK we just crossed the threshold, this term should + // now be "pulsed" into the main postings codec: + postingsDocsWriter.startTerm(); + if (PostingsCodec.DEBUG) + System.out.println(" now flush buffer"); + for(int i=0;i doc); + + return doc; + } + } + } +} + +/** Returned when someone asks for positions() enum on field + * with omitTf true */ +class FakePositionsEnum extends PositionsEnum { + int next() { + return 0; + } + int getPayloadLength() { + return 0; + } + boolean hasPayload() { + return false; + } + byte[] getPayload(byte[] data, int offset) { + return null; + } + } Index: src/java/org/apache/lucene/index/FormatSepDocsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatSepDocsWriter.java (revision 0) +++ src/java/org/apache/lucene/index/FormatSepDocsWriter.java (revision 0) @@ -0,0 +1,230 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; + +/** Writes frq to .frq, docs to .doc, pos to .pos, paylaods + * to .pyl, skip data inlined into terms dict */ + +final class FormatSepDocsWriter extends DocsConsumer { + final static String CODEC = "SepDocFreqSkip"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final IndexOutput freqOut; + final IndexOutput docOut; + final IndexOutput skipOut; + IndexOutput termsOut; + + final FormatSepPositionsWriter posWriter; + final SepSkipListWriter skipListWriter; + final int skipInterval; + final int maxSkipLevels; + final int totalNumDocs; + + boolean omitTF; + boolean storePayloads; + + // Starts a new term + long lastFreqStart; + long freqStart; + long lastDocStart; + long docStart; + long lastSkipStart; + + FieldInfo fieldInfo; + + FormatSepDocsWriter(SegmentWriteState state) throws IOException { + super(); + + final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, "frq"); + state.flushedFiles.add(frqFileName); + freqOut = state.directory.createOutput(frqFileName); + + final String docFileName = IndexFileNames.segmentFileName(state.segmentName, "doc"); + state.flushedFiles.add(docFileName); + docOut = state.directory.createOutput(docFileName); + + final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, "skp"); + state.flushedFiles.add(skipFileName); + skipOut = state.directory.createOutput(skipFileName); + + if (PostingsCodec.DEBUG) + System.out.println("dw.init: create frq=" + frqFileName + " doc=" + docFileName + " skip=" + skipFileName); + + totalNumDocs = state.numDocs; + + // nocommit -- abstraction violation + skipListWriter = new SepSkipListWriter(state.skipInterval, + state.maxSkipLevels, + state.numDocs, + freqOut, docOut, + null, null); + + skipInterval = state.skipInterval; + maxSkipLevels = state.maxSkipLevels; + + posWriter = new FormatSepPositionsWriter(state, this); + } + + void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + PostingsCodec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + // nocommit -- just ask skipper to "start" here + termsOut.writeInt(skipInterval); // write skipInterval + termsOut.writeInt(maxSkipLevels); // write maxSkipLevels + posWriter.start(termsOut); + } + + void startTerm() { + freqStart = freqOut.getFilePointer(); + docStart = docOut.getFilePointer(); + if (!omitTF) + posWriter.startTerm(); + skipListWriter.resetSkip(); + } + + // nocommit -- should we NOT reuse across fields? would + // be cleaner + + // Currently, this instance is re-used across fields, so + // our parent calls setField whenever the field changes + void setField(FieldInfo fieldInfo) { + this.fieldInfo = fieldInfo; + omitTF = fieldInfo.omitTermFreqAndPositions; + storePayloads = fieldInfo.storePayloads; + posWriter.setField(fieldInfo); + } + + int lastDocID; + int df; + + int count; + + /** Adds a new doc in this term. If this returns null + * then we just skip consuming positions/payloads. */ + PositionsConsumer addDoc(int docID, int termDocFreq) throws IOException { + + final int delta = docID - lastDocID; + + if (PostingsCodec.DEBUG) + System.out.println(" dw.addDoc [" + desc + "] count=" + (count++) + " docID=" + docID + " lastDocID=" + lastDocID + " delta=" + delta + " omitTF=" + omitTF + " freq=" + termDocFreq + " freqPointer=" + freqOut.getFilePointer()); + + if (docID < 0 || (df > 0 && delta <= 0)) + throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); + + if ((++df % skipInterval) == 0) { + // TODO: abstraction violation + // nocommit -- awkward we have to make these two + // separate calls to skipper + skipListWriter.setSkipData(lastDocID, storePayloads, posWriter.lastPayloadLength); + skipListWriter.bufferSkip(df); + if (PostingsCodec.DEBUG) + System.out.println(" bufferSkip lastDocID=" + lastDocID + " df=" + df + " freqFP=" + freqOut.getFilePointer() + " docFP=" + docOut.getFilePointer() + " posFP=" + skipListWriter.posOutput.getFilePointer() + " payloadFP=" + skipListWriter.payloadOutput.getFilePointer() + " payloadLen=" + posWriter.lastPayloadLength); + } + + // nocommit -- move this assert up above; every consumer + // shouldn't have to check for this bug: + assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; + + lastDocID = docID; + docOut.writeVInt(delta); + if (!omitTF) + freqOut.writeVInt(termDocFreq); + + // nocommit + if (PostingsCodec.DEBUG) + ((FormatSepPositionsWriter) posWriter).desc = desc + ":" + docID; + + if (omitTF) + return null; + else + return posWriter; + } + + /** Called when we are done adding docs to this term */ + void finishTerm(int docCount, boolean isIndexTerm) throws IOException { + + long skipPos = skipOut.getFilePointer(); + + // nocommit -- wasteful we are counting this in two places? + assert docCount == df; + if (PostingsCodec.DEBUG) + System.out.println("dw.finishTerm termsFP=" + termsOut.getFilePointer() + " freqStart=" + freqStart + " df=" + df + " skipPos=" + skipPos); + + if (isIndexTerm) { + // Write absolute at seek points + termsOut.writeVLong(freqStart); + termsOut.writeVLong(docStart); + } else { + // Write delta between seek points + termsOut.writeVLong(freqStart - lastFreqStart); + termsOut.writeVLong(docStart - lastDocStart); + } + + if (df >= skipInterval) { + if (PostingsCodec.DEBUG) + System.out.println(" writeSkip @ docFp=" + docOut.getFilePointer() + " freqFP=" + freqOut.getFilePointer() + " freqStartFP=" + freqStart + " skipPos=" + skipPos + " lastSkipPos=" + lastSkipStart); + + skipListWriter.writeSkip(skipOut); + } + + if (isIndexTerm) { + termsOut.writeVLong(skipPos); + lastSkipStart = skipPos; + } else if (df >= skipInterval) { + termsOut.writeVLong(skipPos-lastSkipStart); + lastSkipStart = skipPos; + } + + lastFreqStart = freqStart; + lastDocStart = docStart; + + if (!omitTF) + posWriter.finishTerm(isIndexTerm); + + lastDocID = 0; + df = 0; + + // nocommit + count = 0; + } + + void close() throws IOException { + if (PostingsCodec.DEBUG) + System.out.println("dw.close freqFP=" + freqOut.getFilePointer() + " docFP=" + docOut.getFilePointer() + " skipFP=" + skipOut.getFilePointer()); + try { + freqOut.close(); + } finally { + try { + docOut.close(); + } finally { + try { + skipOut.close(); + } finally { + posWriter.close(); + } + } + } + } +} Index: src/java/org/apache/lucene/index/FormatSepPositionsReader.java =================================================================== --- src/java/org/apache/lucene/index/FormatSepPositionsReader.java (revision 0) +++ src/java/org/apache/lucene/index/FormatSepPositionsReader.java (revision 0) @@ -0,0 +1,270 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.util.BitVector; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.Directory; + +// nocommit -- base class should not be named terms dict: +// this class interacts w/ a docsreader +class FormatSepPositionsReader extends FormatPostingsTermsDictPositionsReader { + + static boolean DEBUG = false; + + final IndexInput posIn; + final IndexInput payloadIn; + IndexInput termsIn; + + FormatSepPositionsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize) throws IOException { + assert segmentInfo.getHasProx(); + boolean success = false; + try { + posIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.PROX_EXTENSION), readBufferSize); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "pyl"), readBufferSize); + success = true; + } finally { + if (!success) + close(); + } + } + + void start(IndexInput termsIn) throws IOException { + this.termsIn = termsIn; + + // nocomit -- move these 2 constants into XXXCodec? + PostingsCodec.checkHeader(termsIn, FormatSepPositionsWriter.CODEC, FormatSepPositionsWriter.VERSION_START); + } + + static void files(SegmentInfo segmentInfo, Collection files) { + if (segmentInfo.getHasProx()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, IndexFileNames.PROX_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, "pyl")); + } + } + + Reader reader(FieldInfo fieldInfo, IndexInput termsIn) { + return new TermsDictReader(termsIn, fieldInfo); + } + + void close() throws IOException { + try { + if (posIn != null) + posIn.close(); + } finally { + if (payloadIn != null) + payloadIn.close(); + } + } + + class TermsDictReader extends Reader { + + final IndexInput termsIn; + final FieldInfo fieldInfo; + long posOffset; + long payloadOffset; + + TermsDictReader(IndexInput termsIn, FieldInfo fieldInfo) { + this.termsIn = termsIn; + this.fieldInfo = fieldInfo; + } + + void readTerm(int docFreq, boolean isIndexTerm) throws IOException { + if (DEBUG) + System.out.println(" pr.readterm termsInPointer=" + termsIn.getFilePointer() + " isIndex=" + isIndexTerm); + if (isIndexTerm) { + posOffset = termsIn.readVLong(); + payloadOffset = termsIn.readVLong(); + } else { + posOffset += termsIn.readVLong(); + payloadOffset += termsIn.readVLong(); + } + if (DEBUG) + System.out.println(" posOffset=" + posOffset + " payloadOffset=" + payloadOffset); + if (positions != null) { + positions.seekPending = true; + positions.skipPosOffset = posOffset; + positions.skipPayloadOffset = payloadOffset; + positions.skipPosCount = 0; + } + } + + void close() throws IOException {} + + SegmentPositionsEnum positions; + + PositionsEnum positions() throws IOException { + + if (positions == null) + // Lazy init + positions = new SegmentPositionsEnum(); + + return positions; + } + + // nocommit -- should we have different reader for + // payload vs no payload? + class SegmentPositionsEnum extends PositionsEnum { + + // nocommit + String desc; + + final IndexInput posIn; + final IndexInput payloadIn; + + final boolean storePayloads; + + boolean seekPending; // True if we must seek before reading next position + boolean payloadPending; // True if we must skip payload beore reading next position + + long skipPosOffset; + long skipPayloadOffset; + int skipPosCount; + + int position; + int payloadLength; + + SegmentPositionsEnum() { + if (DEBUG) + System.out.println("new pos enum"); + posIn = (IndexInput) FormatSepPositionsReader.this.posIn.clone(); + storePayloads = fieldInfo.storePayloads; + if (storePayloads) + payloadIn = (IndexInput) FormatSepPositionsReader.this.payloadIn.clone(); + else + payloadIn = null; + } + + void skip(long posOffset, long payloadOffset, int lastPayloadLength, int numPositions) { + skipPosOffset = posOffset; + skipPayloadOffset = payloadOffset; + payloadLength = lastPayloadLength; + assert payloadLength >= 0 || payloadLength == -1; + skipPosCount = numPositions; + seekPending = true; + payloadPending = false; + if (DEBUG) + System.out.println("pr [" + desc + "] skip posFP= " + posOffset + " payloadFP=" + payloadOffset + " numPositions=" + numPositions); + } + + void skip(int numPositions) { + skipPosCount += numPositions; + if (DEBUG) + System.out.println("pr [" + desc + "] skip " + numPositions + " positions; now " + skipPosCount); + } + + void catchUp(int currentCount) throws IOException { + if (DEBUG) + System.out.println(" pos catchup: seekPending=" + seekPending + " skipPosFP=" + skipPosOffset + " skipPayloadFP=" + skipPayloadOffset + " skipPosCount " + skipPosCount + " vs currentCount " + currentCount); + if (seekPending) { + posIn.seek(skipPosOffset); + if (storePayloads) + payloadIn.seek(skipPayloadOffset); + seekPending = false; + } + + while(skipPosCount > currentCount) + next(); + if (DEBUG) + System.out.println(" pos catchup done"); + positions.init(); + } + + void init() { + if (DEBUG) + System.out.println(" pos init"); + position = 0; + } + + int next() throws IOException { + + if (DEBUG) + System.out.println(" pr.next [" + desc + "]: posFP=" + posIn.getFilePointer() + " return pos=" + position); + + final int code = posIn.readVInt(); + + if (storePayloads) { + + if (payloadPending && payloadLength > 0) { + if (DEBUG) + System.out.println(" payload pending: skip " + payloadLength + " bytes"); + // TODO: we could do this lazily, when + // getPayload() is called + payloadIn.seek(payloadIn.getFilePointer()+payloadLength); + } + + if ((code & 1) != 0) { + // Payload length has changed + payloadLength = posIn.readVInt(); + assert payloadLength >= 0; + if (DEBUG) + System.out.println(" new payloadLen=" + payloadLength); + } + assert payloadLength != -1; + + payloadPending = true; + position += code >>> 1; + } else + position += code; + + skipPosCount--; + + // NOTE: the old API actually allowed this... + assert skipPosCount >= 0: "next() was called too many times (more than FormatPostingsDocsEnum.freq() times)"; + + if (DEBUG) + System.out.println(" proxFP=" + posIn.getFilePointer() + " return pos=" + position); + + return position; + } + + int getPayloadLength() { + return payloadLength; + } + + byte[] getPayload(byte[] data, int offset) throws IOException { + + if (!payloadPending) + throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); + + final byte[] retArray; + final int retOffset; + if (data == null || data.length-offset < payloadLength) { + // the array is too small to store the payload data, + // so we allocate a new one + retArray = new byte[payloadLength]; + retOffset = 0; + } else { + retArray = data; + retOffset = offset; + } + + payloadIn.readBytes(retArray, retOffset, payloadLength); + payloadPending = false; + return retArray; + } + + public boolean hasPayload() { + return payloadPending && payloadLength > 0; + } + } + } +} \ No newline at end of file Index: src/java/org/apache/lucene/index/FormatSepPositionsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FormatSepPositionsWriter.java (revision 0) +++ src/java/org/apache/lucene/index/FormatSepPositionsWriter.java (revision 0) @@ -0,0 +1,155 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; + +final class FormatSepPositionsWriter extends PositionsConsumer { + + final static String CODEC = "SepPositionsPayloads"; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final FormatSepDocsWriter parent; + final IndexOutput posOut; + final IndexOutput payloadOut; + + IndexOutput termsOut; + + boolean omitTF; + boolean storePayloads; + int lastPayloadLength = -1; + + // nocommit + String desc; + + FormatSepPositionsWriter(SegmentWriteState state, FormatSepDocsWriter parent) throws IOException { + this.parent = parent; + omitTF = parent.omitTF; + if (state.fieldInfos.hasProx()) { + // At least one field does not omit TF, so create the + // prox file + final String posFileName = IndexFileNames.segmentFileName(state.segmentName, IndexFileNames.PROX_EXTENSION); + state.flushedFiles.add(posFileName); + posOut = state.directory.createOutput(posFileName); + + // nocommit -- only if at least one field stores payloads? + final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, "pyl"); + state.flushedFiles.add(payloadFileName); + payloadOut = state.directory.createOutput(payloadFileName); + + parent.skipListWriter.setPosOutput(posOut); + parent.skipListWriter.setPayloadOutput(payloadOut); + } else { + // Every field omits TF so we will write no prox file + posOut = null; + payloadOut = null; + } + } + + void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + PostingsCodec.writeHeader(termsOut, CODEC, VERSION_CURRENT); + } + + long posStart; + long lastPosStart; + long payloadStart; + long lastPayloadStart; + + void startTerm() { + posStart = posOut.getFilePointer(); + payloadStart = payloadOut.getFilePointer(); + lastPayloadLength = -1; + } + + int lastPosition; + + /** Add a new position & payload */ + void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException { + assert !omitTF: "omitTF is true"; + assert posOut != null; + if (PostingsCodec.DEBUG) + if (payload != null) + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " posFP=" + posOut.getFilePointer() + " payloadFP=" + payloadOut.getFilePointer() + " payload=" + payloadLength + " bytes"); + else + System.out.println("pw.addPos [" + desc + "]: pos=" + position + " posFP=" + posOut.getFilePointer() + " payloadFP=" + payloadOut.getFilePointer()); + + final int delta = position - lastPosition; + lastPosition = position; + + if (storePayloads) { + if (PostingsCodec.DEBUG) + System.out.println(" store payloads"); + if (payloadLength != lastPayloadLength) { + if (PostingsCodec.DEBUG) + System.out.println(" payload len change old=" + lastPayloadLength + " new=" + payloadLength); + lastPayloadLength = payloadLength; + posOut.writeVInt((delta<<1)|1); + posOut.writeVInt(payloadLength); + } else + posOut.writeVInt(delta << 1); + if (payloadLength > 0) + payloadOut.writeBytes(payload, payloadLength); + } else + posOut.writeVInt(delta); + } + + void setField(FieldInfo fieldInfo) { + omitTF = fieldInfo.omitTermFreqAndPositions; + storePayloads = omitTF ? false : fieldInfo.storePayloads; + } + + /** Called when we are done adding positions & payloads */ + void finishDoc() { + lastPosition = 0; + } + + void finishTerm(boolean isIndexTerm) throws IOException { + assert !omitTF; + + if (PostingsCodec.DEBUG) + System.out.println("poswriter finishTerm isIndex=" + isIndexTerm + " posStart=" + posStart + " pointer=" + termsOut.getFilePointer()); + + if (isIndexTerm) { + // Write absolute at seek points + termsOut.writeVLong(posStart); + termsOut.writeVLong(payloadStart); + } else { + termsOut.writeVLong(posStart-lastPosStart); + termsOut.writeVLong(payloadStart-lastPayloadStart); + } + + lastPosStart = posStart; + lastPayloadStart = payloadStart; + } + + void close() throws IOException { + try { + if (posOut != null) + posOut.close(); + } finally { + if (payloadOut != null) + payloadOut.close(); + } + } +} Index: src/java/org/apache/lucene/index/FreqProxTermsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/FreqProxTermsWriter.java (working copy) @@ -88,21 +88,23 @@ } } + final int numAllFields = allFields.size(); + // Sort by field name Collections.sort(allFields); - final int numAllFields = allFields.size(); - // TODO: allow Lucene user to customize this consumer: - final FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos); + // TODO: allow Lucene user to customize this codec: + final FieldsConsumer consumer = state.codec.fieldsConsumer(state); + /* Current writer chain: - FormatPostingsFieldsConsumer - -> IMPL: FormatPostingsFieldsWriter - -> FormatPostingsTermsConsumer - -> IMPL: FormatPostingsTermsWriter - -> FormatPostingsDocConsumer - -> IMPL: FormatPostingsDocWriter - -> FormatPostingsPositionsConsumer + FieldsConsumer + -> IMPL: FormatPostingsTermsDictWriter + -> TermsConsumer + -> IMPL: FormatPostingsTermsDictWriter.TermsWriter + -> DocsConsumer + -> IMPL: FormatPostingsDocsWriter + -> PositionsConsumer -> IMPL: FormatPostingsPositionsWriter */ @@ -145,8 +147,7 @@ FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread) entry.getKey(); perThread.termsHashPerThread.reset(true); } - - consumer.finish(); + consumer.close(); } private byte[] payloadBuffer; @@ -155,7 +156,7 @@ * instances) found in this field and serialize them * into a single RAM segment. */ void appendPostings(FreqProxTermsWriterPerField[] fields, - FormatPostingsFieldsConsumer consumer) + FieldsConsumer consumer) throws CorruptIndexException, IOException { int numFields = fields.length; @@ -172,7 +173,7 @@ assert result; } - final FormatPostingsTermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo); + final TermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo); FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields]; @@ -196,11 +197,15 @@ termStates[numToMerge++] = mergeStates[i]; } - final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(termStates[0].text, termStates[0].textOffset); + final char[] termText = termStates[0].text; + final int termTextOffset = termStates[0].textOffset; + final DocsConsumer docConsumer = termsConsumer.startTerm(termText, termTextOffset); + // Now termStates has numToMerge FieldMergeStates // which all share the same term. Now we must // interleave the docID streams. + int numDocs = 0; while(numToMerge > 0) { FreqProxFieldMergeState minState = termStates[0]; @@ -209,8 +214,9 @@ minState = termStates[i]; final int termDocFreq = minState.termFreq; + numDocs++; - final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq); + final PositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq); final ByteSliceReader prox = minState.prox; @@ -241,7 +247,7 @@ posConsumer.addPosition(position, payloadBuffer, 0, payloadLength); } //End for - posConsumer.finish(); + posConsumer.finishDoc(); } if (!minState.nextDoc()) { @@ -269,7 +275,7 @@ } } - docConsumer.finish(); + termsConsumer.finishTerm(termText, termTextOffset, numDocs); } termsConsumer.finish(); Index: src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 803321) +++ src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -129,7 +129,8 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter) + public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter, + PostingsCodecs codecs) throws CorruptIndexException, IOException { this.docWriter = docWriter; @@ -170,7 +171,7 @@ } SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, fileName); + sis.read(directory, fileName, codecs); } catch (FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -207,7 +208,7 @@ // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, segmentInfos.getCurrentSegmentFileName()); + sis.read(directory, segmentInfos.getCurrentSegmentFileName(), codecs); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } Index: src/java/org/apache/lucene/index/IndexFileNames.java =================================================================== --- src/java/org/apache/lucene/index/IndexFileNames.java (revision 803321) +++ src/java/org/apache/lucene/index/IndexFileNames.java (working copy) @@ -109,6 +109,10 @@ GEN_EXTENSION, NORMS_EXTENSION, COMPOUND_FILE_STORE_EXTENSION, + // nocommit -- need cleaner way! + "doc", + "pyl", + "skp" }; /** File extensions that are added to a compound file @@ -154,6 +158,12 @@ TERMS_INDEX_EXTENSION, TERMS_EXTENSION }; + + static final String COMPOUND_EXTENSIONS_NOT_CODEC[] = new String[] { + FIELD_INFOS_EXTENSION, + FIELDS_INDEX_EXTENSION, + FIELDS_EXTENSION, + }; /** File extensions for term vector support */ static final String VECTOR_EXTENSIONS[] = new String[] { Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 803321) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -238,7 +238,7 @@ final Directory dir = FSDirectory.getDirectory(path); IndexReader r = null; try { - r = open(dir, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + r = open(dir, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } finally { if (r == null) dir.close(); @@ -275,7 +275,7 @@ final Directory dir = FSDirectory.getDirectory(path); IndexReader r = null; try { - r = open(dir, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + r = open(dir, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } finally { if (r == null) dir.close(); @@ -292,7 +292,7 @@ * Use {@link #open(Directory, boolean)} instead */ public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException { - return open(directory, null, null, false, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, null, null, false, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Returns an IndexReader reading the index in the given @@ -306,7 +306,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns a read/write IndexReader reading the index in the given @@ -318,7 +318,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), null, commit, false, DEFAULT_TERMS_INDEX_DIVISOR); + return open(commit.getDirectory(), null, commit, false, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in the given @@ -332,7 +332,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns a read/write IndexReader reading the index in the given @@ -347,7 +347,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, false, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, deletionPolicy, null, false, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in @@ -365,7 +365,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in @@ -393,7 +393,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor); + return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, null); } /** Expert: returns a read/write IndexReader reading the index in the given @@ -411,7 +411,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, false, DEFAULT_TERMS_INDEX_DIVISOR); + return open(commit.getDirectory(), deletionPolicy, commit, false, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in @@ -431,7 +431,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in @@ -461,11 +461,14 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); } - private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor); + private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor, + PostingsCodecs codecs) throws CorruptIndexException, IOException { + if (codecs == null) + codecs = PostingsCodecs.getDefault(); + return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); } /** @@ -679,7 +682,7 @@ * @throws IOException if there is a low-level IO error */ public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentVersion(directory); + return SegmentInfos.readCurrentVersion(directory, PostingsCodecs.getDefault()); } /** @@ -697,7 +700,7 @@ * @see #getCommitUserData() */ public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentUserData(directory); + return SegmentInfos.readCurrentUserData(directory, PostingsCodecs.getDefault()); } /** @@ -1026,15 +1029,22 @@ * calling terms(), {@link TermEnum#next()} must be called * on the resulting enumeration before calling other methods such as * {@link TermEnum#term()}. + * @deprecated Use {@link #fields()} instead. * @throws IOException if there is a low-level IO error */ public abstract TermEnum terms() throws IOException; + // Default impl emulates new API using old one + public FieldsEnum fields() throws IOException { + return new LegacyFieldsEnum(this); + } + /** Returns an enumeration of all terms starting at a given term. If * the given term does not exist, the enumeration is positioned at the * first term greater than the supplied term. The enumeration is * ordered by Term.compareTo(). Each term is greater than all that * precede it in the enumeration. + * @deprecated nocommit * @throws IOException if there is a low-level IO error */ public abstract TermEnum terms(Term t) throws IOException; @@ -1055,6 +1065,7 @@ * *

The enumeration is ordered by document number. Each document number * is greater than all that precede it in the enumeration. + * @deprecated Use {@link #termDocsEnum(Term)} instead. * @throws IOException if there is a low-level IO error */ public TermDocs termDocs(Term term) throws IOException { @@ -1064,7 +1075,49 @@ return termDocs; } + private static class NullDocsEnum extends DocsEnum { + public int skipTo(int target) { + return -1; + } + public int next() { + return -1; + } + public int freq() { + return 1; + } + public int ord() { + return 0; + } + public int read(int[] docs, int[] freqs) { + return 0; + } + public PositionsEnum positions() { + return null; + } + public void close() { + } + } + private static final NullDocsEnum nullDocsEnum = new NullDocsEnum(); + + /** Returns DocsEnum for this term, or null if the field + * or term text do not exist in the index. */ + public DocsEnum termDocsEnum(Term term) throws IOException { + + // nocommit -- not good, because we don't close up the + // chain when docsEnum.close is called? + FieldsEnum fields = fields(); + if (fields.seek(term.field())) { + TermsEnum terms = fields.terms(); + if (terms.seek(term.text())) { + return terms.docs(); + } + } + return nullDocsEnum; + } + + /** Returns an unpositioned {@link TermDocs} enumerator. + * @deprecated nocommit * @throws IOException if there is a low-level IO error */ public abstract TermDocs termDocs() throws IOException; @@ -1084,6 +1137,7 @@ *

This positional information facilitates phrase and proximity searching. *

The enumeration is ordered by document number. Each document number is * greater than all that precede it in the enumeration. + * @deprecated nocommit * @throws IOException if there is a low-level IO error */ public TermPositions termPositions(Term term) throws IOException { @@ -1094,6 +1148,7 @@ } /** Returns an unpositioned {@link TermPositions} enumerator. + * @deprecated nocommit * @throws IOException if there is a low-level IO error */ public abstract TermPositions termPositions() throws IOException; Index: src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/IndexWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -466,7 +466,7 @@ // reader; in theory we could do similar retry logic, // just like we do when loading segments_N synchronized(this) { - return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor); + return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor, codecs); } } @@ -944,7 +944,7 @@ */ public IndexWriter(String path, Analyzer a, boolean create, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(FSDirectory.getDirectory(path), a, create, true, null, false, mfl.getLimit(), null, null); + init(FSDirectory.getDirectory(path), a, create, true, null, false, mfl.getLimit(), null, null, null); } /** @@ -973,7 +973,7 @@ */ public IndexWriter(String path, Analyzer a, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException { - init(FSDirectory.getDirectory(path), a, create, true, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(FSDirectory.getDirectory(path), a, create, true, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } /** @@ -1006,7 +1006,7 @@ */ public IndexWriter(File path, Analyzer a, boolean create, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(FSDirectory.getDirectory(path), a, create, true, null, false, mfl.getLimit(), null, null); + init(FSDirectory.getDirectory(path), a, create, true, null, false, mfl.getLimit(), null, null, null); } /** @@ -1035,7 +1035,7 @@ */ public IndexWriter(File path, Analyzer a, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException { - init(FSDirectory.getDirectory(path), a, create, true, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(FSDirectory.getDirectory(path), a, create, true, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } /** @@ -1066,7 +1066,7 @@ */ public IndexWriter(Directory d, Analyzer a, boolean create, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, false, null, false, mfl.getLimit(), null, null); + init(d, a, create, false, null, false, mfl.getLimit(), null, null, null); } /** @@ -1094,7 +1094,7 @@ */ public IndexWriter(Directory d, Analyzer a, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, false, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, create, false, null, true, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } /** @@ -1307,7 +1307,7 @@ */ public IndexWriter(Directory d, boolean autoCommit, Analyzer a, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, false, null, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, create, false, null, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } /** @@ -1394,7 +1394,7 @@ */ public IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, false, deletionPolicy, false, mfl.getLimit(), null, null); + init(d, a, create, false, deletionPolicy, false, mfl.getLimit(), null, null, null); } /** @@ -1429,9 +1429,9 @@ * false or if there is any other low-level * IO error */ - IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexingChain indexingChain, IndexCommit commit) + IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexingChain indexingChain, IndexCommit commit, PostingsCodecs codecs) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, false, deletionPolicy, false, mfl.getLimit(), indexingChain, commit); + init(d, a, create, false, deletionPolicy, false, mfl.getLimit(), indexingChain, commit, codecs); } /** @@ -1464,7 +1464,7 @@ */ public IndexWriter(Directory d, boolean autoCommit, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, create, false, deletionPolicy, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null); + init(d, a, create, false, deletionPolicy, autoCommit, DEFAULT_MAX_FIELD_LENGTH, null, null, null); } /** @@ -1505,24 +1505,30 @@ */ public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexCommit commit) throws CorruptIndexException, LockObtainFailedException, IOException { - init(d, a, false, false, deletionPolicy, false, mfl.getLimit(), null, commit); + init(d, a, false, false, deletionPolicy, false, mfl.getLimit(), null, commit, null); } + + PostingsCodecs codecs; private void init(Directory d, Analyzer a, boolean closeDir, IndexDeletionPolicy deletionPolicy, boolean autoCommit, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit) throws CorruptIndexException, LockObtainFailedException, IOException { if (IndexReader.indexExists(d)) { - init(d, a, false, closeDir, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit); + init(d, a, false, closeDir, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit, null); } else { - init(d, a, true, closeDir, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit); + init(d, a, true, closeDir, deletionPolicy, autoCommit, maxFieldLength, indexingChain, commit, null); } } private void init(Directory d, Analyzer a, final boolean create, boolean closeDir, IndexDeletionPolicy deletionPolicy, boolean autoCommit, int maxFieldLength, - IndexingChain indexingChain, IndexCommit commit) + IndexingChain indexingChain, IndexCommit commit, PostingsCodecs codecsIn) throws CorruptIndexException, LockObtainFailedException, IOException { this.closeDir = closeDir; + if (codecs == null) + codecs = PostingsCodecs.getDefault(); + else + codecs = codecsIn; directory = d; analyzer = a; setMessageID(defaultInfoStream); @@ -1550,6 +1556,7 @@ boolean doCommit; try { segmentInfos.read(directory); + segmentInfos.read(directory, codecs); segmentInfos.clear(); doCommit = false; } catch (IOException e) { @@ -1569,7 +1576,7 @@ changeCount++; } } else { - segmentInfos.read(directory); + segmentInfos.read(directory, codecs); if (commit != null) { // Swap out all segments, but, keep metadata in @@ -1580,7 +1587,7 @@ if (commit.getDirectory() != directory) throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory"); SegmentInfos oldInfos = new SegmentInfos(); - oldInfos.read(directory, commit.getSegmentsFileName()); + oldInfos.read(directory, commit.getSegmentsFileName(), codecs); segmentInfos.replace(oldInfos); changeCount++; if (infoStream != null) @@ -1603,7 +1610,7 @@ // KeepOnlyLastCommitDeleter: deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, infoStream, docWriter); + segmentInfos, infoStream, docWriter, this.codecs); if (deleter.startingCommitDeleted) // Deletion policy deleted the "head" commit point. @@ -3542,7 +3549,7 @@ ensureOpen(); for (int i = 0; i < dirs.length; i++) { SegmentInfos sis = new SegmentInfos(); // read infos from dir - sis.read(dirs[i]); + sis.read(dirs[i], codecs); for (int j = 0; j < sis.size(); j++) { final SegmentInfo info = sis.info(j); docCount += info.docCount; @@ -3672,7 +3679,7 @@ } SegmentInfos sis = new SegmentInfos(); // read infos from dir - sis.read(dirs[i]); + sis.read(dirs[i], codecs); for (int j = 0; j < sis.size(); j++) { SegmentInfo info = sis.info(j); assert !segmentInfos.contains(info): "dup info dir=" + info.dir + " name=" + info.name; @@ -3855,10 +3862,11 @@ // call hits an exception it will release the write // lock: startTransaction(true); - + success = false; + try { mergedName = newSegmentName(); - merger = new SegmentMerger(this, mergedName, null); + merger = new SegmentMerger(this, mergedName, null, codecs); SegmentReader sReader = null; synchronized(this) { @@ -3881,7 +3889,7 @@ synchronized(this) { segmentInfos.clear(); // pop old infos & add new info = new SegmentInfo(mergedName, docCount, directory, false, true, - -1, null, false, merger.hasProx()); + -1, null, false, merger.hasProx(), merger.getCodec()); setDiagnostics(info, "addIndexes(IndexReader[])"); segmentInfos.add(info); } @@ -3928,7 +3936,7 @@ startTransaction(false); try { - merger.createCompoundFile(mergedName + ".cfs"); + merger.createCompoundFile(mergedName + ".cfs", info); synchronized(this) { info.setUseCompoundFile(true); } @@ -4304,7 +4312,9 @@ directory, false, true, docStoreOffset, docStoreSegment, docStoreIsCompoundFile, - docWriter.hasProx()); + docWriter.hasProx(), + docWriter.getCodec()); + setDiagnostics(newSegment, "flush"); } @@ -4821,7 +4831,8 @@ docStoreOffset, docStoreSegment, docStoreIsCompoundFile, - false); + false, + null); Map details = new HashMap(); @@ -4939,7 +4950,7 @@ if (infoStream != null) message("merging " + merge.segString(directory)); - merger = new SegmentMerger(this, mergedName, merge); + merger = new SegmentMerger(this, mergedName, merge, codecs); merge.readers = new SegmentReader[numSegments]; merge.readersClone = new SegmentReader[numSegments]; @@ -5012,6 +5023,9 @@ // This is where all the work happens: mergedDocCount = merge.info.docCount = merger.merge(merge.mergeDocStores); + // Record which codec was used to write the segment + merge.info.setCodec(merger.getCodec()); + assert mergedDocCount == totDocCount; // TODO: in the non-realtime case, we may want to only @@ -5095,7 +5109,7 @@ final String compoundFileName = mergedName + "." + IndexFileNames.COMPOUND_FILE_EXTENSION; try { - merger.createCompoundFile(compoundFileName); + merger.createCompoundFile(compoundFileName, merge.info); success = true; } catch (IOException ioe) { synchronized(this) { Index: src/java/org/apache/lucene/index/LegacyFieldsEnum.java =================================================================== --- src/java/org/apache/lucene/index/LegacyFieldsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/LegacyFieldsEnum.java (revision 0) @@ -0,0 +1,192 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.util.BitVector; + +/** Implements new API (FieldsEnum/TermsEnum) on top of old + * API. Used only for IndexReader impls outside Lucene's + * core. */ +class LegacyFieldsEnum extends FieldsEnum { + private final IndexReader r; + private TermEnum terms; + private String field; + + public LegacyFieldsEnum(IndexReader r) throws IOException { + this.r = r; + terms = r.terms(); + } + + private void doSeek(Term t) throws IOException { + terms.close(); + terms = r.terms(t); + } + + public boolean seek(String field) throws IOException { + this.field = field; + doSeek(new Term(field, "")); + return terms.term() != null && terms.term().field.equals(field); + } + + public boolean next() throws IOException { + + final Term seekTo = new Term(field, "\uFFFF"); + + doSeek(seekTo); + if (terms.term() != null) { + String newField = terms.term().field; + assert !newField.equals(field); + field = newField; + return true; + } else + return false; + } + + public TermsEnum terms() throws IOException { + return new LegacyTermsEnum(); + } + + public String field() { + return field; + } + + public void close() throws IOException { + terms.close(); + } + + private class LegacyTermsEnum extends TermsEnum { + + private TermEnum terms; + + LegacyTermsEnum() throws IOException { + this.terms = r.terms(new Term(field, "")); + } + + public boolean seek(String text) throws IOException { + terms.close(); + terms = r.terms(new Term(field, text)); + // assert terms.term().field.equals(field); + return terms.term() != null && terms.term().text.equals(text); + } + + public boolean next() throws IOException { + return terms.next(); + } + + public String text() { + return terms.term().text; + } + + public int docFreq() { + return terms.docFreq(); + } + + public long ord() { + throw new UnsupportedOperationException(); + } + + public DocsEnum docs() throws IOException { + return new LegacyDocsEnum(terms.term()); + } + + public void close() throws IOException { + terms.close(); + } + } + + private class LegacyDocsEnum extends DocsEnum { + final TermDocs td; + final Term term; + + TermPositions tp; + + LegacyDocsEnum(Term term) throws IOException { + this.term = term; + td = r.termDocs(term); + } + + public int next() throws IOException { + if (td.next()) { + return td.doc(); + } else { + return -1; + } + } + + public int skipTo(int target) throws IOException { + if (td.skipTo(target)) { + return td.doc(); + } else { + return -1; + } + } + + public int freq() { + return td.freq(); + } + + public int ord() { + throw new UnsupportedOperationException(); + } + + public int read(int[] docs, int[] freqs) throws IOException { + return td.read(docs, freqs); + } + + public void close() throws IOException { + td.close(); + } + + LegacyPositionsEnum lpe; + + public PositionsEnum positions() throws IOException { + if (tp == null) { + tp = r.termPositions(term); + lpe = new LegacyPositionsEnum(tp); + } else { + tp.seek(term); + } + return lpe; + } + } + + private class LegacyPositionsEnum extends PositionsEnum { + + final TermPositions tp; + LegacyPositionsEnum(TermPositions tp) { + this.tp = tp; + } + + public int next() throws IOException { + return tp.nextPosition(); + } + + public int getPayloadLength() { + return tp.getPayloadLength(); + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + return tp.getPayload(data, offset); + } + + public boolean hasPayload() { + return tp.isPayloadAvailable(); + } + } +} \ No newline at end of file Index: src/java/org/apache/lucene/index/MultiLevelSkipListWriter.java =================================================================== --- src/java/org/apache/lucene/index/MultiLevelSkipListWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/MultiLevelSkipListWriter.java (working copy) @@ -134,6 +134,7 @@ */ long writeSkip(IndexOutput output) throws IOException { long skipPointer = output.getFilePointer(); + //System.out.println("skipper.writeSkip fp=" + skipPointer); if (skipBuffer == null || skipBuffer.length == 0) return skipPointer; for (int level = numberOfSkipLevels - 1; level > 0; level--) { Index: src/java/org/apache/lucene/index/ParallelReader.java =================================================================== --- src/java/org/apache/lucene/index/ParallelReader.java (revision 803321) +++ src/java/org/apache/lucene/index/ParallelReader.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.util.BitVector; import java.io.IOException; import java.util.*; @@ -122,6 +123,68 @@ } decrefOnClose.add(Boolean.valueOf(incRefReaders)); } + + private class ParallelFieldsEnum extends FieldsEnum { + String currentField; + IndexReader currentReader; + Iterator keys; + private final HashMap readerFields = new HashMap(); + + public boolean seek(String field) throws IOException { + currentField = field; + currentReader = (IndexReader) fieldToReader.get(currentField); + keys = null; + return currentReader != null; + } + + public boolean next() throws IOException { + if (keys == null) { + // nocommit -- not right after seek -- must advance + // iterator + keys = fieldToReader.keySet().iterator(); + } + if (keys.hasNext()) { + currentField = (String) keys.next(); + currentReader = (IndexReader) fieldToReader.get(currentField); + return true; + } else { + currentField = null; + return false; + } + } + + private FieldsEnum getReaderFields() throws IOException { + FieldsEnum fields = (FieldsEnum) readerFields.get(currentReader); + if (fields == null ) { + fields = currentReader.fields(); + readerFields.put(currentReader, fields); + } + boolean result = fields.seek(currentField); + assert result; + return fields; + } + + public String field() { + return currentField; + } + + public TermsEnum terms() throws IOException { + if (currentReader != null) { + FieldsEnum fields = getReaderFields(); + return fields.terms(); + } else { + return null; + } + } + + public void close() { + // nocommit -- must close all readerFields + } + } + + public FieldsEnum fields() { + return new ParallelFieldsEnum(); + } public synchronized Object clone() { try { Index: src/java/org/apache/lucene/index/PositionsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/PositionsConsumer.java (revision 0) +++ src/java/org/apache/lucene/index/PositionsConsumer.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; + +abstract class PositionsConsumer { + + abstract void start(IndexOutput termsOut) throws IOException; + + abstract void startTerm() throws IOException; + + /** Add a new position & payload. If payloadLength > 0 + * you must read those bytes from the IndexInput. NOTE: + * you must fully consume the byte[] payload, since + * caller is free to reuse it on subsequent calls. */ + abstract void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException; + + /** Called when we are done adding positions & payloads + * for each doc */ + abstract void finishDoc() throws IOException; + + abstract void finishTerm(boolean isIndexTerm) throws IOException; + + abstract void close() throws IOException; +} Index: src/java/org/apache/lucene/index/PositionsEnum.java =================================================================== --- src/java/org/apache/lucene/index/PositionsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/PositionsEnum.java (revision 0) @@ -0,0 +1,38 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.AttributeSource; + +public abstract class PositionsEnum extends AttributeSource { + + /** Returns the next position. You should only call this + * up to {@link FormatPostingsDocsEnum#freq()} times else + * the behavior is not defined. */ + abstract int next() throws IOException; + + abstract int getPayloadLength(); + + // nocommit -- improve this so that readers that do their + // own buffering can save a copy + abstract byte[] getPayload(byte[] data, int offset) throws IOException; + + abstract boolean hasPayload(); +} Index: src/java/org/apache/lucene/index/PostingsCodec.java =================================================================== --- src/java/org/apache/lucene/index/PostingsCodec.java (revision 0) +++ src/java/org/apache/lucene/index/PostingsCodec.java (revision 0) @@ -0,0 +1,67 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.io.IOException; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +abstract class PostingsCodec { + + static boolean DEBUG = false; + + static final int CODEC_HEADER = 0x1af65; + + /** Unique name that's used to retrieve this codec when + * reading the index */ + String name; + + /** Writes a new segment */ + abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException; + + /** Reads a segment */ + abstract FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException; + + /** Gathers files associated with this segment */ + abstract void files(SegmentInfo segmentInfo, Collection files); + + static void checkHeader(IndexInput in, String codec, int version) throws IOException { + + // Safety to guard against reading a bogus string: + int header = in.readVInt(); + if (header != CODEC_HEADER) + throw new CorruptIndexException("codec header mismatch"); + + final String actualCodec = in.readString(); + if (!codec.equals(actualCodec)) + throw new CorruptIndexException("codec mismatch: expected '" + codec + "' but got '" + actualCodec + "'"); + + int actualVersion = in.readVInt(); + if (actualVersion > version) + throw new CorruptIndexException("version '" + actualVersion + "' is too new (expected <= '" + version + "'"); + } + + static void writeHeader(IndexOutput out, String codec, int version) throws IOException { + out.writeVInt(CODEC_HEADER); + out.writeString(codec); + out.writeVInt(version); + } +} Index: src/java/org/apache/lucene/index/PostingsCodecs.java =================================================================== --- src/java/org/apache/lucene/index/PostingsCodecs.java (revision 0) +++ src/java/org/apache/lucene/index/PostingsCodecs.java (revision 0) @@ -0,0 +1,70 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.HashMap; + +/** Holds a set of codecs, keyed by name. You subclass + * this, instantiate it, and register your codecs, then + * pass this instance to IndexReader/IndexWriter (via + * package private APIs) to use different codecs when + * reading & writing segments. */ + +abstract class PostingsCodecs { + + private final HashMap codecs = new HashMap(); + + void register(PostingsCodec codec) { + if (codec.name == null) + throw new IllegalArgumentException("code.name is null"); + if (!codecs.containsKey(codec.name)) { + codecs.put(codec.name, codec); + } else if (codecs.get(codec.name) != codec) + throw new IllegalArgumentException("codec '" + codec.name + "' is already registered as a different codec instance"); + } + + PostingsCodec lookup(String name) { + final PostingsCodec codec = (PostingsCodec) codecs.get(name); + if (codec == null) + throw new IllegalArgumentException("required codec '" + name + "' not found"); + return codec; + } + + abstract PostingsCodec getWriter(SegmentWriteState state); + + static private final PostingsCodecs defaultCodecs = new DefaultPostingsCodecs(); + + static PostingsCodecs getDefault() { + return defaultCodecs; + } +} + +class DefaultPostingsCodecs extends PostingsCodecs { + DefaultPostingsCodecs() { + register(new DefaultCodec()); + register(new PreFlexCodec()); + register(new PulsingCodec()); + register(new SepCodec()); + } + + PostingsCodec getWriter(SegmentWriteState state) { + return lookup("Default"); + //return lookup("Pulsing"); + //return lookup("Sep"); + } +} \ No newline at end of file Index: src/java/org/apache/lucene/index/PreFlexCodec.java =================================================================== --- src/java/org/apache/lucene/index/PreFlexCodec.java (revision 0) +++ src/java/org/apache/lucene/index/PreFlexCodec.java (revision 0) @@ -0,0 +1,45 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.io.IOException; + +import org.apache.lucene.store.Directory; + +/** Codec that reads the pre-flex-indexing postings + * format. It does not provide a writer because newly + * written segments should use DefaultCodec. */ +class PreFlexCodec extends PostingsCodec { + + PreFlexCodec() { + name = "PreFlex"; + } + + FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + throw new IllegalArgumentException("this codec can only be used for reading"); + } + + FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor) throws IOException { + return new PreFlexTermInfosReader(dir, fieldInfos, info, readBufferSize, indexDivisor); + } + + void files(SegmentInfo info, Collection files) { + PreFlexTermInfosReader.files(info, files); + } +} Index: src/java/org/apache/lucene/index/PreFlexTermInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/PreFlexTermInfosReader.java (revision 0) +++ src/java/org/apache/lucene/index/PreFlexTermInfosReader.java (revision 0) @@ -0,0 +1,244 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.Collection; +import java.util.Iterator; + +import org.apache.lucene.util.BitVector; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.Directory; + +class PreFlexTermInfosReader extends FieldsProducer { + + final TermInfosReader tis; + final IndexInput freqStream; + final IndexInput proxStream; + final private FieldInfos fieldInfos; + final TreeMap fields = new TreeMap(); /*String -> FieldInfo */ + + PreFlexTermInfosReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor) + throws IOException { + tis = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor); + this.fieldInfos = fieldInfos; + + // make sure that all index files have been read or are kept open + // so that if an index update removes them we'll still have them + freqStream = dir.openInput(info.name + ".frq", readBufferSize); + boolean anyProx = false; + final int numFields = fieldInfos.size(); + for(int i=0;i 0; i--) { // read segmentInfos - add(new SegmentInfo(directory, format, input)); + add(new SegmentInfo(directory, format, input, codecs)); } if(format >= 0){ // in old format the version number may be at the end of the file @@ -300,13 +305,16 @@ * @throws IOException if there is a low-level IO error */ public final void read(Directory directory) throws CorruptIndexException, IOException { - + read(directory, PostingsCodecs.getDefault()); + } + + public final void read(Directory directory, final PostingsCodecs codecs) throws CorruptIndexException, IOException { generation = lastGeneration = -1; new FindSegmentsFile(directory) { protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - read(directory, segmentFileName); + read(directory, segmentFileName, codecs); return null; } }.run(); @@ -372,6 +380,8 @@ public Object clone() { SegmentInfos sis = (SegmentInfos) super.clone(); for(int i=0;i 0) { int matchSize = 0; // pop matching terms @@ -642,6 +668,9 @@ termsConsumer = consumer.addField(fieldInfo); omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions; } + + if (PostingsCodec.DEBUG) + System.out.println("merge term=" + term); int df = appendPostings(termsConsumer, match, matchSize); // add new TermInfo @@ -666,6 +695,8 @@ int[] getDelCounts() { return delCounts; } + + private char[] termBuffer; /** Process postings from multiple segments all positioned on the * same term. Writes out merged entries into freqOutput and @@ -677,10 +708,18 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int appendPostings(final FormatPostingsTermsConsumer termsConsumer, SegmentMergeInfo[] smis, int n) + private final int appendPostings(final TermsConsumer termsConsumer, SegmentMergeInfo[] smis, int n) throws CorruptIndexException, IOException { - final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(smis[0].term.text); + final String text = smis[0].term.text; + final int len = text.length(); + if (termBuffer == null || termBuffer.length < 1+len) + termBuffer = new char[ArrayUtil.getNextSize(1+len)]; + text.getChars(0, len, termBuffer, 0); + termBuffer[len] = 0xffff; + + final DocsConsumer docConsumer = termsConsumer.startTerm(termBuffer, 0); + int df = 0; for (int i = 0; i < n; i++) { SegmentMergeInfo smi = smis[i]; @@ -693,13 +732,20 @@ while (postings.next()) { df++; int doc = postings.doc(); - if (docMap != null) + if (docMap != null) { doc = docMap[doc]; // map around deletions + assert doc != -1: "postings enum returned deleted docID " + postings.doc() + " freq=" + postings.freq() + " df=" + df; + } + doc += base; // convert to merged space final int freq = postings.freq(); - final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(doc, freq); + final PositionsConsumer posConsumer = docConsumer.addDoc(doc, freq); + // nocommit -- omitTF should be "private", and this + // code (and FreqProxTermsWriter) should instead + // check if posConsumer is null? + if (!omitTermFreqAndPositions) { for (int j = 0; j < freq; j++) { final int position = postings.nextPosition(); @@ -711,11 +757,11 @@ } posConsumer.addPosition(position, payloadBuffer, 0, payloadLength); } - posConsumer.finish(); + posConsumer.finishDoc(); } } } - docConsumer.finish(); + termsConsumer.finishTerm(termBuffer, 0, df); return df; } Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 803321) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -37,6 +37,8 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BitVector; import org.apache.lucene.util.CloseableThreadLocal; +import org.apache.lucene.util.cache.Cache; +import org.apache.lucene.util.cache.SimpleLRUCache; /** @version $Id */ /** @@ -48,6 +50,7 @@ private SegmentInfo si; private int readBufferSize; + boolean isPreFlex; CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); @@ -83,23 +86,29 @@ final String segment; final FieldInfos fieldInfos; - final IndexInput freqStream; - final IndexInput proxStream; - final TermInfosReader tisNoIndex; + //final IndexInput freqStream; + //final IndexInput proxStream; + //final TermInfosReader tisNoIndex; + FieldsProducer terms; + boolean isPreFlex; + PostingsCodecs codecs; + final Directory dir; final Directory cfsDir; final int readBufferSize; final int termsIndexDivisor; - TermInfosReader tis; FieldsReader fieldsReaderOrig; TermVectorsReader termVectorsReaderOrig; CompoundFileReader cfsReader; CompoundFileReader storeCFSReader; - CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException { + CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, PostingsCodecs codecs) throws IOException { segment = si.name; + if (codecs == null) + codecs = PostingsCodecs.getDefault(); + this.codecs = codecs; this.readBufferSize = readBufferSize; this.dir = dir; @@ -116,23 +125,12 @@ fieldInfos = new FieldInfos(cfsDir, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION); this.termsIndexDivisor = termsIndexDivisor; - TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor); - if (termsIndexDivisor == -1) { - tisNoIndex = reader; - } else { - tis = reader; - tisNoIndex = null; - } - // make sure that all index files have been read or are kept open - // so that if an index update removes them we'll still have them - freqStream = cfsDir.openInput(segment + "." + IndexFileNames.FREQ_EXTENSION, readBufferSize); + // Ask codec for its FieldsProducer + terms = si.getCodec().fieldsProducer(cfsDir, fieldInfos, si, readBufferSize, termsIndexDivisor); + assert terms != null; - if (fieldInfos.hasProx()) { - proxStream = cfsDir.openInput(segment + "." + IndexFileNames.PROX_EXTENSION, readBufferSize); - } else { - proxStream = null; - } + isPreFlex = terms instanceof PreFlexTermInfosReader; success = true; } finally { if (!success) { @@ -157,66 +155,46 @@ return cfsReader; } - synchronized TermInfosReader getTermsReader() { - if (tis != null) { - return tis; - } else { - return tisNoIndex; - } - } - + // nocommit synchronized boolean termsIndexIsLoaded() { - return tis != null; + return true; + //return tis != null; } + // nocommit // NOTE: only called from IndexWriter when a near // real-time reader is opened, or applyDeletes is run, // sharing a segment that's still being merged. This // method is not fully thread safe, and relies on the // synchronization in IndexWriter synchronized void loadTermsIndex(SegmentInfo si, int termsIndexDivisor) throws IOException { - if (tis == null) { - Directory dir0; - if (si.getUseCompoundFile()) { - // In some cases, we were originally opened when CFS - // was not used, but then we are asked to open the - // terms reader with index, the segment has switched - // to CFS - if (cfsReader == null) { - cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize); - } - dir0 = cfsReader; - } else { - dir0 = dir; - } - - tis = new TermInfosReader(dir0, segment, fieldInfos, readBufferSize, termsIndexDivisor); - } +// if (tis == null) { +// Directory dir0; +// if (si.getUseCompoundFile()) { +// // In some cases, we were originally opened when CFS +// // was not used, but then we are asked to open the +// // terms reader with index, the segment has switched +// // to CFS +// if (cfsReader == null) { +// cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize); +// } +// dir0 = cfsReader; +// } else { +// dir0 = dir; +// } +// +// tis = new TermInfosReader(dir0, segment, fieldInfos, readBufferSize, termsIndexDivisor); +// } } synchronized void decRef() throws IOException { if (ref.decRef() == 0) { - // close everything, nothing is shared anymore with other readers - if (tis != null) { - tis.close(); - // null so if an app hangs on to us we still free most ram - tis = null; + if (terms != null) { + terms.close(); } - - if (tisNoIndex != null) { - tisNoIndex.close(); - } - - if (freqStream != null) { - freqStream.close(); - } - if (proxStream != null) { - proxStream.close(); - } - if (termVectorsReaderOrig != null) { termVectorsReaderOrig.close(); } @@ -588,7 +566,7 @@ * @deprecated */ public static SegmentReader get(SegmentInfo si) throws CorruptIndexException, IOException { - return get(false, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + return get(false, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null); } /** @@ -596,7 +574,7 @@ * @throws IOException if there is a low-level IO error */ public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor); + return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor, null); } /** @@ -605,7 +583,7 @@ * @deprecated */ static SegmentReader get(SegmentInfo si, int readBufferSize, boolean doOpenStores, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return get(false, si.dir, si, readBufferSize, doOpenStores, termInfosIndexDivisor); + return get(false, si.dir, si, readBufferSize, doOpenStores, termInfosIndexDivisor, null); } /** @@ -617,8 +595,12 @@ SegmentInfo si, int readBufferSize, boolean doOpenStores, - int termInfosIndexDivisor) + int termInfosIndexDivisor, + PostingsCodecs codecs) throws CorruptIndexException, IOException { + if (codecs == null) + codecs = PostingsCodecs.getDefault(); + SegmentReader instance; try { if (readOnly) @@ -635,7 +617,7 @@ boolean success = false; try { - instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor); + instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor, codecs); if (doOpenStores) { instance.core.openDocStores(si); } @@ -929,14 +911,32 @@ return new ArrayList(si.files()); } + /** @deprecated nocommit */ public TermEnum terms() { ensureOpen(); - return core.getTermsReader().terms(); + if (isPreFlex) { + // For old API on an old segment, instead of + // converting old API -> new API -> old API, just give + // direct access to old: + return ((PreFlexTermInfosReader) core.terms).tis.terms(); + } else + // Emulate old API + return new TermsDictTermEnum(); } public TermEnum terms(Term t) throws IOException { ensureOpen(); - return core.getTermsReader().terms(t); + if (isPreFlex) + // For old API on an old segment, instead of + // converting old API -> new API -> old API, just give + // direct access to old: + return ((PreFlexTermInfosReader) core.terms).tis.terms(t); + else { + // Emulate old API + TermsDictTermEnum terms = new TermsDictTermEnum(); + terms.seek(t); + return terms; + } } FieldInfos fieldInfos() { @@ -952,6 +952,7 @@ return (deletedDocs != null && deletedDocs.get(n)); } + /** @deprecated nocommit */ public TermDocs termDocs(Term term) throws IOException { if (term == null) { return new AllTermDocs(this); @@ -959,24 +960,88 @@ return super.termDocs(term); } } + + public FieldsEnum fields() throws IOException { + return core.terms.fields(deletedDocs); + } + /** @deprecated nocommit */ public TermDocs termDocs() throws IOException { ensureOpen(); - return new SegmentTermDocs(this); + if (isPreFlex) { + // For old API on an old segment, instead of + // converting old API -> new API -> old API, just give + // direct access to old: + final PreFlexTermInfosReader pre = (PreFlexTermInfosReader) core.terms; + return new SegmentTermDocs(pre.freqStream, deletedDocs, pre.tis, core.fieldInfos); + } else + // Emulate old API + return new TermsDictTermDocs(); } public TermPositions termPositions() throws IOException { ensureOpen(); - return new SegmentTermPositions(this); + if (isPreFlex) { + final PreFlexTermInfosReader pre = (PreFlexTermInfosReader) core.terms; + return new SegmentTermPositions(pre.freqStream, pre.proxStream, deletedDocs, pre.tis, core.fieldInfos); + } else + // Emulate old API + return new TermsDictTermPositions(); } + private final CloseableThreadLocal perThread = new CloseableThreadLocal(); + + // nocommit -- move term vectors under here + private static final class PerThread { + TermsDictTermEnum terms; + + // Used for caching the least recently looked-up Terms + Cache termsCache; + } + + private final static int DEFAULT_TERMS_CACHE_SIZE = 1024; + + private PerThread getPerThread() { + PerThread resources = (PerThread) perThread.get(); + if (resources == null) { + resources = new PerThread(); + resources.terms = new TermsDictTermEnum(); + // Cache does not have to be thread-safe, it is only used by one thread at the same time + resources.termsCache = new SimpleLRUCache(DEFAULT_TERMS_CACHE_SIZE); + perThread.set(resources); + } + return resources; + } + + public int docFreq(Term t) throws IOException { ensureOpen(); - TermInfo ti = core.getTermsReader().get(t); - if (ti != null) - return ti.docFreq; - else - return 0; + //if (tis == null) { + PerThread thread = getPerThread(); + Integer result = (Integer) thread.termsCache.get(t); + if (result == null) { + // Cache miss + final int freq; + thread.terms.seek(t); + if (thread.terms.term() != null && thread.terms.term().equals(t)) { + freq = thread.terms.docFreq(); + } else + freq = 0; + result = new Integer(freq); + thread.termsCache.put(t, result); + } + + return result.intValue(); + + /* + } else { + TermInfo ti = tis.get(t); + if (ti != null) + return ti.docFreq; + else + return 0; + } + */ } public int numDocs() { @@ -1323,13 +1388,15 @@ // This is necessary so that cloned SegmentReaders (which // share the underlying postings data) will map to the // same entry in the FieldCache. See LUCENE-1579. + // nocommit - what to return here? public final Object getFieldCacheKey() { - return core.freqStream; + return core.terms; } - public long getUniqueTermCount() { - return core.getTermsReader().size(); - } + // nocommit +// public long getUniqueTermCount() { +// return core.terms.size(); +// } /** * Lotsa tests did hacks like:
@@ -1358,4 +1425,302 @@ public int getTermInfosIndexDivisor() { return core.termsIndexDivisor; } + + // Back compat: implements TermEnum API using new flex API + final private class TermsDictTermEnum extends TermEnum { + FieldsEnum fields; + TermsEnum terms; + boolean done; + + public boolean next() throws IOException { + + if (PostingsCodec.DEBUG) + System.out.println("tdte.next done=" + done + " seg=" + core.segment); + + if (done) + return false; + + if (fields == null) { + fields = core.terms.fields(deletedDocs); + if (!fields.next()) { + if (PostingsCodec.DEBUG) + System.out.println(" fields.next returned false"); + done = true; + return false; + } else + terms = fields.terms(); + } + + while(true) { + if (terms == null) { + // Advance to the next field + if (done) + return false; + if (!fields.next()) { + done = true; + return false; + } + terms = fields.terms(); + } + if (terms.next()) + // This field still has terms + return true; + else { + // Done producing terms from this field + //terms.close(); + terms = null; + } + } + } + + public Term term() { + if (terms != null && !done) { + final String text = terms.text(); + if (text != null) + return new Term(fields.field(), text); + } + return null; + } + + public int docFreq() { + if (terms == null) + return 0; + else + return terms.docFreq(); + } + + public void close() throws IOException { + if (terms != null) { + //terms.close(); + terms = null; + } + if (fields != null) { + //fields.close(); + fields = null; + } + } + + // Seek forward only + /* + public boolean skipTo(Term target) throws IOException { + // Just use seek, if the target is beyond our current + // point, else next(): + + if (done) + // Already EOF + return false; + + if (terms != null) { + + final int cmp = target.field.compareTo(fields.fieldInfo().name); + if (cmp < 0) + // Target is before our current term + return next(); + else if (cmp == 0) { + final int cmp2 = target.text.compareTo(terms.text()); + if (cmp2 < 0) + // Target is before our current term + return next(); + } + } + + // OK target is in the future, so just seek + return seek(target); + } + */ + + // nocommit -- technically this method only need work on + // a newly created term enum? ie i can move into ctor? + public boolean seek(Term target) throws IOException { + + if (terms == null || !fields.field().equals(target.field)) { + + // Seek field + if (terms != null) { + //terms.close(); + terms = null; + } + + if (fields == null) { + fields = core.terms.fields(deletedDocs); + if (!fields.next()) { + done = true; + return false; + } + } + + if (fields.seek(target.field)) { + // Field matches + terms = fields.terms(); + assert fields.field().equals(target.field); + assert terms != null; + } else + // nocommit -- not right + //return next(); + return true; + } + + // Field matches; now seek text + terms.seek(target.text); + return terms.text() != null; + } + } + + // Back compat + private class TermsDictTermDocs implements TermDocs { + + FieldsEnum fields; + TermsEnum terms; + DocsEnum docs; + int doc; + + TermsDictTermDocs() throws IOException { + fields = core.terms.fields(deletedDocs); + } + + public void close() throws IOException { + if (docs != null) { + //docs.close(); + docs = null; + } + if (terms != null) { + //terms.close(); + terms = null; + } + if (fields != null) { + //fields.close(); + fields = null; + } + } + + public void seek(TermEnum termEnum) throws IOException { + // nocommit -- optimize for the special cases here + seek(termEnum.term()); + } + + public boolean skipTo(int target) throws IOException { + if (docs == null) return false; + doc = docs.skipTo(target); + return doc != -1; + } + + public int read(int[] docs, int[] freqs) throws IOException { + if (this.docs == null) return 0; + return this.docs.read(docs, freqs); + } + + public void seek(Term term) throws IOException { + + if (PostingsCodec.DEBUG) + System.out.println("\nwrapper termdocs.seek term=" + term); + + if (docs != null) { + //docs.close(); + docs = null; + } + + if (terms != null && !term.field.equals(fields.field())) { + if (PostingsCodec.DEBUG) + System.out.println(" switch field"); + if (terms != null) { + //terms.close(); + terms = null; + } + } + + if (terms == null) + if (fields.seek(term.field)) { + // Field exists; get the terms + terms = fields.terms(); + assert terms != null; + } else + // Field does not exist + return; + + // Seek + if (terms.seek(term.text)) + // Term exists + docs = terms.docs(); + } + + public int doc() { + if (docs == null) return 0; + else return doc; + } + + public int freq() { + if (docs == null) return 0; + return docs.freq(); + } + + public boolean next() throws IOException { + if (docs == null) return false; + doc = docs.next(); + return doc != -1; + } + } + + // Back compat + final private class TermsDictTermPositions extends TermsDictTermDocs implements TermPositions { + + PositionsEnum positions; + + TermsDictTermPositions() throws IOException { + super(); + } + + public void seek(TermEnum termEnum) throws IOException { + super.seek(termEnum); + if (docs != null) + positions = docs.positions(); + } + + public boolean skipTo(int target) throws IOException { + boolean result = super.skipTo(target); + if (result && docs != null) + positions = docs.positions(); + else + positions = null; + return result; + } + + public int read(int[] docs, int[] freqs) throws IOException { + throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead."); + } + + public void seek(Term term) throws IOException { + super.seek(term); + if (docs != null) + positions = docs.positions(); + else + positions = null; + } + + public boolean next() throws IOException { + boolean result = super.next(); + if (result && docs != null) + positions = docs.positions(); + else + positions = null; + return result; + } + + public int nextPosition() throws IOException { + return positions.next(); + } + + public int getPayloadLength() { + return positions.getPayloadLength(); + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + return positions.getPayload(data, offset); + } + + public boolean isPayloadAvailable() { + return positions.hasPayload(); + } + } + + } Index: src/java/org/apache/lucene/index/SegmentTermDocs.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermDocs.java (revision 803321) +++ src/java/org/apache/lucene/index/SegmentTermDocs.java (working copy) @@ -22,11 +22,13 @@ import org.apache.lucene.store.IndexInput; class SegmentTermDocs implements TermDocs { - protected SegmentReader parent; + //protected SegmentReader parent; + protected final BitVector deletedDocs; + private final FieldInfos fieldInfos; + private final TermInfosReader tis; protected IndexInput freqStream; protected int count; protected int df; - protected BitVector deletedDocs; int doc = 0; int freq; @@ -43,6 +45,7 @@ protected boolean currentFieldStoresPayloads; protected boolean currentFieldOmitTermFreqAndPositions; + /* protected SegmentTermDocs(SegmentReader parent) { this.parent = parent; this.freqStream = (IndexInput) parent.core.freqStream.clone(); @@ -52,9 +55,19 @@ this.skipInterval = parent.core.getTermsReader().getSkipInterval(); this.maxSkipLevels = parent.core.getTermsReader().getMaxSkipLevels(); } + */ + + protected SegmentTermDocs(IndexInput freqStream, BitVector deletedDocs, TermInfosReader tis, FieldInfos fieldInfos) { + this.freqStream = (IndexInput) freqStream.clone(); + this.deletedDocs = deletedDocs; + this.tis = tis; + this.fieldInfos = fieldInfos; + skipInterval = tis.getSkipInterval(); + maxSkipLevels = tis.getMaxSkipLevels(); + } public void seek(Term term) throws IOException { - TermInfo ti = parent.core.getTermsReader().get(term); + TermInfo ti = tis.get(term); seek(ti, term); } @@ -63,13 +76,13 @@ Term term; // use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs - if (termEnum instanceof SegmentTermEnum && ((SegmentTermEnum) termEnum).fieldInfos == parent.core.fieldInfos) { // optimized case + if (termEnum instanceof SegmentTermEnum && ((SegmentTermEnum) termEnum).fieldInfos == fieldInfos) { // optimized case SegmentTermEnum segmentTermEnum = ((SegmentTermEnum) termEnum); term = segmentTermEnum.term(); ti = segmentTermEnum.termInfo(); } else { // punt case term = termEnum.term(); - ti = parent.core.getTermsReader().get(term); + ti = tis.get(term); } seek(ti, term); @@ -77,7 +90,7 @@ void seek(TermInfo ti, Term term) throws IOException { count = 0; - FieldInfo fi = parent.core.fieldInfos.fieldInfo(term.field); + FieldInfo fi = fieldInfos.fieldInfo(term.field); currentFieldOmitTermFreqAndPositions = (fi != null) ? fi.omitTermFreqAndPositions : false; currentFieldStoresPayloads = (fi != null) ? fi.storePayloads : false; if (ti == null) { Index: src/java/org/apache/lucene/index/SegmentTermEnum.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermEnum.java (revision 803321) +++ src/java/org/apache/lucene/index/SegmentTermEnum.java (working copy) @@ -20,6 +20,10 @@ import java.io.IOException; import org.apache.lucene.store.IndexInput; +/** + * @deprecated No longer used with flex indexing, except for + * reading old segments */ + final class SegmentTermEnum extends TermEnum implements Cloneable { private IndexInput input; FieldInfos fieldInfos; Index: src/java/org/apache/lucene/index/SegmentTermPositions.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermPositions.java (revision 803321) +++ src/java/org/apache/lucene/index/SegmentTermPositions.java (working copy) @@ -18,12 +18,14 @@ */ import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitVector; import java.io.IOException; final class SegmentTermPositions extends SegmentTermDocs implements TermPositions { private IndexInput proxStream; + private IndexInput proxStreamOrig; private int proxCount; private int position; @@ -37,11 +39,18 @@ // for a lazy skip private long lazySkipPointer = -1; private int lazySkipProxCount = 0; - + + /* SegmentTermPositions(SegmentReader p) { super(p); this.proxStream = null; // the proxStream will be cloned lazily when nextPosition() is called for the first time } + */ + + SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, BitVector deletedDocs, TermInfosReader tis, FieldInfos fieldInfos) { + super(freqStream, deletedDocs, tis, fieldInfos); + this.proxStreamOrig = proxStream; // the proxStream will be cloned lazily when nextPosition() is called for the first time + } final void seek(TermInfo ti, Term term) throws IOException { super.seek(ti, term); @@ -146,7 +155,7 @@ private void lazySkip() throws IOException { if (proxStream == null) { // clone lazily - proxStream = (IndexInput) parent.core.proxStream.clone(); + proxStream = (IndexInput)proxStreamOrig.clone(); } // we might have to skip the current payload Index: src/java/org/apache/lucene/index/SegmentWriteState.java =================================================================== --- src/java/org/apache/lucene/index/SegmentWriteState.java (revision 803321) +++ src/java/org/apache/lucene/index/SegmentWriteState.java (working copy) @@ -26,21 +26,47 @@ DocumentsWriter docWriter; Directory directory; String segmentName; + FieldInfos fieldInfos; String docStoreSegmentName; int numDocs; - int termIndexInterval; int numDocsInStore; Collection flushedFiles; - public SegmentWriteState(DocumentsWriter docWriter, Directory directory, String segmentName, String docStoreSegmentName, int numDocs, - int numDocsInStore, int termIndexInterval) { + // Actual codec used + PostingsCodec codec; + + /** Expert: The fraction of terms in the "dictionary" which should be stored + * in RAM. Smaller values use more memory, but make searching slightly + * faster, while larger values use less memory and make searching slightly + * slower. Searching is typically not dominated by dictionary lookup, so + * tweaking this is rarely useful.*/ + int termIndexInterval; + + /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, + * used to accellerate {@link TermDocs#skipTo(int)}. Larger values result in + * smaller indexes, greater acceleration, but fewer accelerable cases, while + * smaller values result in bigger indexes, less acceleration and more + * accelerable cases. More detailed experiments would be useful here. */ + int skipInterval = 16; + + /** Expert: The maximum number of skip levels. Smaller values result in + * slightly smaller indexes, but slower skipping in big posting lists. + */ + int maxSkipLevels = 10; + + public SegmentWriteState(DocumentsWriter docWriter, Directory directory, String segmentName, FieldInfos fieldInfos, + String docStoreSegmentName, int numDocs, + int numDocsInStore, int termIndexInterval, + PostingsCodecs codecs) { this.docWriter = docWriter; this.directory = directory; this.segmentName = segmentName; + this.fieldInfos = fieldInfos; this.docStoreSegmentName = docStoreSegmentName; this.numDocs = numDocs; this.numDocsInStore = numDocsInStore; this.termIndexInterval = termIndexInterval; + this.codec = codecs.getWriter(this); flushedFiles = new HashSet(); } Index: src/java/org/apache/lucene/index/SepCodec.java =================================================================== --- src/java/org/apache/lucene/index/SepCodec.java (revision 0) +++ src/java/org/apache/lucene/index/SepCodec.java (revision 0) @@ -0,0 +1,63 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.io.IOException; + +import org.apache.lucene.store.Directory; + +class SepCodec extends PostingsCodec { + + SepCodec() { + name = "Sep"; + } + + FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + DocsConsumer docsWriter = new FormatSepDocsWriter(state); + boolean success = false; + try { + FieldsConsumer ret = new FormatPostingsTermsDictWriter(state, docsWriter); + success = true; + return ret; + } finally { + if (!success) + docsWriter.close(); + } + } + + FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException { + FormatPostingsTermsDictDocsReader docsReader = new FormatSepDocsReader(dir, si, readBufferSize); + boolean success = false; + try { + FieldsProducer ret = new FormatPostingsTermsDictReader(dir, fieldInfos, si.name, + docsReader, + readBufferSize, indexDivisor); + success = true; + return ret; + } finally { + if (!success) + docsReader.close(); + } + } + + void files(SegmentInfo segmentInfo, Collection files) { + FormatSepDocsReader.files(segmentInfo, files); + FormatPostingsTermsDictReader.files(segmentInfo, files); + } +} Index: src/java/org/apache/lucene/index/SepSkipListReader.java =================================================================== --- src/java/org/apache/lucene/index/SepSkipListReader.java (revision 0) +++ src/java/org/apache/lucene/index/SepSkipListReader.java (revision 0) @@ -0,0 +1,140 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexInput; + +/** + * Implements the skip list reader for the default posting list format + * that stores positions and payloads. + * + */ +class SepSkipListReader extends MultiLevelSkipListReader { + private boolean currentFieldStoresPayloads; + private long freqPointer[]; + private long docPointer[]; + private long posPointer[]; + private long payloadPointer[]; + private int payloadLength[]; + + private long lastFreqPointer; + private long lastDocPointer; + private long lastPosPointer; + private long lastPayloadPointer; + private int lastPayloadLength; + + + SepSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) { + super(skipStream, maxSkipLevels, skipInterval); + freqPointer = new long[maxSkipLevels]; + docPointer = new long[maxSkipLevels]; + posPointer = new long[maxSkipLevels]; + payloadPointer = new long[maxSkipLevels]; + payloadLength = new int[maxSkipLevels]; + } + + void init(long skipPointer, long docBasePointer, long freqBasePointer, long posBasePointer, long payloadBasePointer, int df, boolean storesPayloads) { + super.init(skipPointer, df); + this.currentFieldStoresPayloads = storesPayloads; + lastFreqPointer = freqBasePointer; + lastDocPointer = docBasePointer; + lastPosPointer = posBasePointer; + lastPayloadPointer = payloadBasePointer; + + Arrays.fill(docPointer, docBasePointer); + Arrays.fill(freqPointer, freqBasePointer); + Arrays.fill(posPointer, posBasePointer); + Arrays.fill(payloadPointer, payloadBasePointer); + Arrays.fill(payloadLength, 0); + } + + /** Returns the freq pointer of the doc to which the last call of + * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ + long getFreqPointer() { + return lastFreqPointer; + } + + long getDocPointer() { + return lastDocPointer; + } + + /** Returns the prox pointer of the doc to which the last call of + * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ + long getPosPointer() { + return lastPosPointer; + } + + long getPayloadPointer() { + return lastPayloadPointer; + } + + /** Returns the payload length of the payload stored just before + * the doc to which the last call of {@link MultiLevelSkipListReader#skipTo(int)} + * has skipped. */ + int getPayloadLength() { + return lastPayloadLength; + } + + protected void seekChild(int level) throws IOException { + super.seekChild(level); + freqPointer[level] = lastFreqPointer; + docPointer[level] = lastDocPointer; + posPointer[level] = lastPosPointer; + payloadPointer[level] = lastPayloadPointer; + payloadLength[level] = lastPayloadLength; + } + + protected void setLastSkipData(int level) { + super.setLastSkipData(level); + lastFreqPointer = freqPointer[level]; + lastDocPointer = docPointer[level]; + lastPosPointer = posPointer[level]; + lastPayloadPointer = payloadPointer[level]; + lastPayloadLength = payloadLength[level]; + } + + + protected int readSkipData(int level, IndexInput skipStream) throws IOException { + int delta; + //System.out.println(" readSkipData skipFP=" + skipStream.getFilePointer() + " storesPayloads=" + currentFieldStoresPayloads); + if (currentFieldStoresPayloads) { + // the current field stores payloads. + // if the doc delta is odd then we have + // to read the current payload length + // because it differs from the length of the + // previous payload + delta = skipStream.readVInt(); + if ((delta & 1) != 0) { + payloadLength[level] = skipStream.readVInt(); + } + delta >>>= 1; + } else { + delta = skipStream.readVInt(); + } + //System.out.println(" delta=" + delta + " level=" + level); + freqPointer[level] += skipStream.readVInt(); + docPointer[level] += skipStream.readVInt(); + posPointer[level] += skipStream.readVInt(); + payloadPointer[level] += skipStream.readVInt(); + + return delta; + } +} Index: src/java/org/apache/lucene/index/SepSkipListWriter.java =================================================================== --- src/java/org/apache/lucene/index/SepSkipListWriter.java (revision 0) +++ src/java/org/apache/lucene/index/SepSkipListWriter.java (revision 0) @@ -0,0 +1,173 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexOutput; + + +/** + * Implements the skip list writer for the default posting list format + * that stores positions and payloads. + * + */ +class SepSkipListWriter extends MultiLevelSkipListWriter { + private int[] lastSkipDoc; + private int[] lastSkipPayloadLength; + private long[] lastSkipDocPointer; + private long[] lastSkipFreqPointer; + private long[] lastSkipPosPointer; + private long[] lastSkipPayloadPointer; + + private IndexOutput freqOutput; + private IndexOutput docOutput; + // nocommit -- private again + IndexOutput posOutput; + // nocommit -- private again + IndexOutput payloadOutput; + + private int curDoc; + private boolean curStorePayloads; + private int curPayloadLength; + private long curFreqPointer; + private long curDocPointer; + private long curPosPointer; + private long curPayloadPointer; + + SepSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, + IndexOutput freqOutput, + IndexOutput docOutput, + IndexOutput posOutput, + IndexOutput payloadOutput) { + super(skipInterval, numberOfSkipLevels, docCount); + + this.freqOutput = freqOutput; + this.docOutput = docOutput; + this.posOutput = posOutput; + this.payloadOutput = payloadOutput; + + lastSkipDoc = new int[numberOfSkipLevels]; + lastSkipPayloadLength = new int[numberOfSkipLevels]; + lastSkipFreqPointer = new long[numberOfSkipLevels]; + lastSkipDocPointer = new long[numberOfSkipLevels]; + lastSkipPosPointer = new long[numberOfSkipLevels]; + lastSkipPayloadPointer = new long[numberOfSkipLevels]; + } + + void setFreqOutput(IndexOutput freqOutput) { + this.freqOutput = freqOutput; + } + + void setDocOutput(IndexOutput docOutput) { + this.docOutput = docOutput; + } + + void setPosOutput(IndexOutput posOutput) { + this.posOutput = posOutput; + } + + void setPayloadOutput(IndexOutput payloadOutput) { + this.payloadOutput = payloadOutput; + } + + /** + * Sets the values for the current skip data. + */ + void setSkipData(int doc, boolean storePayloads, int payloadLength) { + this.curDoc = doc; + this.curStorePayloads = storePayloads; + this.curPayloadLength = payloadLength; + this.curFreqPointer = freqOutput.getFilePointer(); + this.curDocPointer = docOutput.getFilePointer(); + if (posOutput != null) + this.curPosPointer = posOutput.getFilePointer(); + if (payloadOutput != null) + this.curPayloadPointer = payloadOutput.getFilePointer(); + } + + protected void resetSkip() { + super.resetSkip(); + Arrays.fill(lastSkipDoc, 0); + Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list + Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); + Arrays.fill(lastSkipDocPointer, docOutput.getFilePointer()); + if (posOutput != null) + Arrays.fill(lastSkipPosPointer, posOutput.getFilePointer()); + if (payloadOutput != null) + Arrays.fill(lastSkipPayloadPointer, payloadOutput.getFilePointer()); + + if (PostingsCodec.DEBUG) + System.out.println(" skip writer base freqFP=" + freqOutput.getFilePointer() + " posFP=" + posOutput.getFilePointer()); + } + + protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { + // To efficiently store payloads in the posting lists we do not store the length of + // every payload. Instead we omit the length for a payload if the previous payload had + // the same length. + // However, in order to support skipping the payload length at every skip point must be known. + // So we use the same length encoding that we use for the posting lists for the skip data as well: + // Case 1: current field does not store payloads + // SkipDatum --> DocSkip, FreqSkip, ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // DocSkip records the document number before every SkipInterval th document in TermFreqs. + // Document numbers are represented as differences from the previous value in the sequence. + // Case 2: current field stores payloads + // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // PayloadLength --> VInt + // In this case DocSkip/2 is the difference between + // the current and the previous value. If DocSkip + // is odd, then a PayloadLength encoded as VInt follows, + // if DocSkip is even, then it is assumed that the + // current payload length equals the length at the previous + // skip point + //System.out.println(" skip writer level=" + level + " curDoc=" + curDoc + " lastDoc=" + lastSkipDoc[level] + " delta=" + (curDoc - lastSkipDoc[level]) + " storePayloads=" + curStorePayloads + " skipBufferFP=" + skipBuffer.getFilePointer()); + if (curStorePayloads) { + int delta = curDoc - lastSkipDoc[level]; + if (curPayloadLength == lastSkipPayloadLength[level]) { + // the current payload length equals the length at the previous skip point, + // so we don't store the length again + skipBuffer.writeVInt(delta << 1); + } else { + // the payload length is different from the previous one. We shift the DocSkip, + // set the lowest bit and store the current payload length as VInt. + skipBuffer.writeVInt(delta << 1 | 1); + skipBuffer.writeVInt(curPayloadLength); + lastSkipPayloadLength[level] = curPayloadLength; + } + } else { + // current field does not store payloads + skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); + } + + // nocommit -- if payloads / pos not stored for this + // field, don't encode these 0's + skipBuffer.writeVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); + skipBuffer.writeVInt((int) (curDocPointer - lastSkipDocPointer[level])); + skipBuffer.writeVInt((int) (curPosPointer - lastSkipPosPointer[level])); + skipBuffer.writeVInt((int) (curPayloadPointer - lastSkipPayloadPointer[level])); + + lastSkipDoc[level] = curDoc; + lastSkipFreqPointer[level] = curFreqPointer; + lastSkipDocPointer[level] = curDocPointer; + lastSkipPosPointer[level] = curPosPointer; + lastSkipPayloadPointer[level] = curPayloadPointer; + } +} Index: src/java/org/apache/lucene/index/TermDocs.java =================================================================== --- src/java/org/apache/lucene/index/TermDocs.java (revision 803321) +++ src/java/org/apache/lucene/index/TermDocs.java (working copy) @@ -26,7 +26,8 @@ ordered by document number. @see IndexReader#termDocs() - */ + @deprecated Use DocsEnum instead +*/ public interface TermDocs { /** Sets this to the data for a term. Index: src/java/org/apache/lucene/index/TermEnum.java =================================================================== --- src/java/org/apache/lucene/index/TermEnum.java (revision 803321) +++ src/java/org/apache/lucene/index/TermEnum.java (working copy) @@ -22,7 +22,8 @@ /** Abstract class for enumerating terms.

Term enumerations are always ordered by Term.compareTo(). Each term in - the enumeration is greater than all that precede it. */ + the enumeration is greater than all that precede it. +* @deprecated Use TermsEnum instead */ public abstract class TermEnum { /** Increments the enumeration to the next element. True if one exists.*/ Index: src/java/org/apache/lucene/index/TermInfo.java =================================================================== --- src/java/org/apache/lucene/index/TermInfo.java (revision 803321) +++ src/java/org/apache/lucene/index/TermInfo.java (working copy) @@ -17,7 +17,10 @@ * limitations under the License. */ -/** A TermInfo is the record of information stored for a term.*/ +/** A TermInfo is the record of information stored for a + * term + * @deprecated This class is no longer used in flexible + * indexing. */ final class TermInfo { /** The number of documents which contain the term. */ Index: src/java/org/apache/lucene/index/TermInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/TermInfosReader.java (revision 803321) +++ src/java/org/apache/lucene/index/TermInfosReader.java (working copy) @@ -26,8 +26,9 @@ /** This stores a monotonically increasing set of pairs in a * Directory. Pairs are accessed either by Term or by ordinal position the - * set. */ - + * set + * @deprecated This class has been replaced by + * FormatPostingsTermsDictReader, except for reading old segments. */ final class TermInfosReader { private final Directory directory; private final String segment; @@ -197,7 +198,10 @@ return ti; } } - + + // nocommit -- make sure these optimizations survive + // into flex + // optimize sequential access: first try scanning cached enum w/o seeking SegmentTermEnum enumerator = resources.termEnum; if (enumerator.term() != null // term is at or past current Index: src/java/org/apache/lucene/index/TermInfosWriter.java =================================================================== --- src/java/org/apache/lucene/index/TermInfosWriter.java (revision 803321) +++ src/java/org/apache/lucene/index/TermInfosWriter.java (working copy) @@ -24,8 +24,10 @@ import org.apache.lucene.util.UnicodeUtil; /** This stores a monotonically increasing set of pairs in a - Directory. A TermInfos can be written once, in order. */ - + Directory. A TermInfos can be written once, in order. + * + * @deprecated This class has been replaced by + * FormatPostingsTermsDictWriter. */ final class TermInfosWriter { /** The file format version, a negative number. */ public static final int FORMAT = -3; @@ -36,193 +38,4 @@ // NOTE: always change this if you switch to a new format! public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - private FieldInfos fieldInfos; - private IndexOutput output; - private TermInfo lastTi = new TermInfo(); - private long size; - - // TODO: the default values for these two parameters should be settable from - // IndexWriter. However, once that's done, folks will start setting them to - // ridiculous values and complaining that things don't work well, as with - // mergeFactor. So, let's wait until a number of folks find that alternate - // values work better. Note that both of these values are stored in the - // segment, so that it's safe to change these w/o rebuilding all indexes. - - /** Expert: The fraction of terms in the "dictionary" which should be stored - * in RAM. Smaller values use more memory, but make searching slightly - * faster, while larger values use less memory and make searching slightly - * slower. Searching is typically not dominated by dictionary lookup, so - * tweaking this is rarely useful.*/ - int indexInterval = 128; - - /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, - * used to accellerate {@link TermDocs#skipTo(int)}. Larger values result in - * smaller indexes, greater acceleration, but fewer accelerable cases, while - * smaller values result in bigger indexes, less acceleration and more - * accelerable cases. More detailed experiments would be useful here. */ - int skipInterval = 16; - - /** Expert: The maximum number of skip levels. Smaller values result in - * slightly smaller indexes, but slower skipping in big posting lists. - */ - int maxSkipLevels = 10; - - private long lastIndexPointer; - private boolean isIndex; - private byte[] lastTermBytes = new byte[10]; - private int lastTermBytesLength = 0; - private int lastFieldNumber = -1; - - private TermInfosWriter other; - private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result(); - - TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval) - throws IOException { - initialize(directory, segment, fis, interval, false); - other = new TermInfosWriter(directory, segment, fis, interval, true); - other.other = this; - } - - private TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval, boolean isIndex) throws IOException { - initialize(directory, segment, fis, interval, isIndex); - } - - private void initialize(Directory directory, String segment, FieldInfos fis, - int interval, boolean isi) throws IOException { - indexInterval = interval; - fieldInfos = fis; - isIndex = isi; - output = directory.createOutput(segment + (isIndex ? ".tii" : ".tis")); - output.writeInt(FORMAT_CURRENT); // write format - output.writeLong(0); // leave space for size - output.writeInt(indexInterval); // write indexInterval - output.writeInt(skipInterval); // write skipInterval - output.writeInt(maxSkipLevels); // write maxSkipLevels - assert initUTF16Results(); - } - - void add(Term term, TermInfo ti) throws IOException { - UnicodeUtil.UTF16toUTF8(term.text, 0, term.text.length(), utf8Result); - add(fieldInfos.fieldNumber(term.field), utf8Result.result, utf8Result.length, ti); - } - - // Currently used only by assert statements - UnicodeUtil.UTF16Result utf16Result1; - UnicodeUtil.UTF16Result utf16Result2; - - // Currently used only by assert statements - private boolean initUTF16Results() { - utf16Result1 = new UnicodeUtil.UTF16Result(); - utf16Result2 = new UnicodeUtil.UTF16Result(); - return true; - } - - // Currently used only by assert statement - private int compareToLastTerm(int fieldNumber, byte[] termBytes, int termBytesLength) { - - if (lastFieldNumber != fieldNumber) { - final int cmp = fieldInfos.fieldName(lastFieldNumber).compareTo(fieldInfos.fieldName(fieldNumber)); - // If there is a field named "" (empty string) then we - // will get 0 on this comparison, yet, it's "OK". But - // it's not OK if two different field numbers map to - // the same name. - if (cmp != 0 || lastFieldNumber != -1) - return cmp; - } - - UnicodeUtil.UTF8toUTF16(lastTermBytes, 0, lastTermBytesLength, utf16Result1); - UnicodeUtil.UTF8toUTF16(termBytes, 0, termBytesLength, utf16Result2); - final int len; - if (utf16Result1.length < utf16Result2.length) - len = utf16Result1.length; - else - len = utf16Result2.length; - - for(int i=0;i, TermInfo> pair to the set. - Term must be lexicographically greater than all previous Terms added. - TermInfo pointers must be positive and greater than all previous.*/ - void add(int fieldNumber, byte[] termBytes, int termBytesLength, TermInfo ti) - throws IOException { - - assert compareToLastTerm(fieldNumber, termBytes, termBytesLength) < 0 || - (isIndex && termBytesLength == 0 && lastTermBytesLength == 0) : - "Terms are out of order: field=" + fieldInfos.fieldName(fieldNumber) + " (number " + fieldNumber + ")" + - " lastField=" + fieldInfos.fieldName(lastFieldNumber) + " (number " + lastFieldNumber + ")" + - " text=" + new String(termBytes, 0, termBytesLength, "UTF-8") + " lastText=" + new String(lastTermBytes, 0, lastTermBytesLength, "UTF-8"); - - assert ti.freqPointer >= lastTi.freqPointer: "freqPointer out of order (" + ti.freqPointer + " < " + lastTi.freqPointer + ")"; - assert ti.proxPointer >= lastTi.proxPointer: "proxPointer out of order (" + ti.proxPointer + " < " + lastTi.proxPointer + ")"; - - if (!isIndex && size % indexInterval == 0) - other.add(lastFieldNumber, lastTermBytes, lastTermBytesLength, lastTi); // add an index term - - writeTerm(fieldNumber, termBytes, termBytesLength); // write term - - output.writeVInt(ti.docFreq); // write doc freq - output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers - output.writeVLong(ti.proxPointer - lastTi.proxPointer); - - if (ti.docFreq >= skipInterval) { - output.writeVInt(ti.skipOffset); - } - - if (isIndex) { - output.writeVLong(other.output.getFilePointer() - lastIndexPointer); - lastIndexPointer = other.output.getFilePointer(); // write pointer - } - - lastFieldNumber = fieldNumber; - lastTi.set(ti); - size++; - } - - private void writeTerm(int fieldNumber, byte[] termBytes, int termBytesLength) - throws IOException { - - // TODO: UTF16toUTF8 could tell us this prefix - // Compute prefix in common with last term: - int start = 0; - final int limit = termBytesLength < lastTermBytesLength ? termBytesLength : lastTermBytesLength; - while(start < limit) { - if (termBytes[start] != lastTermBytes[start]) - break; - start++; - } - - final int length = termBytesLength - start; - output.writeVInt(start); // write shared prefix length - output.writeVInt(length); // write delta length - output.writeBytes(termBytes, start, length); // write delta bytes - output.writeVInt(fieldNumber); // write field num - if (lastTermBytes.length < termBytesLength) { - byte[] newArray = new byte[(int) (termBytesLength*1.5)]; - System.arraycopy(lastTermBytes, 0, newArray, 0, start); - lastTermBytes = newArray; - } - System.arraycopy(termBytes, start, lastTermBytes, start, length); - lastTermBytesLength = termBytesLength; - } - - /** Called to complete TermInfos creation. */ - void close() throws IOException { - output.seek(4); // write size after format - output.writeLong(size); - output.close(); - - if (!isIndex) - other.close(); - } - } Index: src/java/org/apache/lucene/index/TermPositions.java =================================================================== --- src/java/org/apache/lucene/index/TermPositions.java (revision 803321) +++ src/java/org/apache/lucene/index/TermPositions.java (working copy) @@ -26,6 +26,7 @@ * positions of each occurrence of a term in a document. * * @see IndexReader#termPositions() + * @deprecated Use PositionsEnum instead */ public interface TermPositions Index: src/java/org/apache/lucene/index/TermsConsumer.java =================================================================== --- src/java/org/apache/lucene/index/TermsConsumer.java (revision 0) +++ src/java/org/apache/lucene/index/TermsConsumer.java (revision 0) @@ -0,0 +1,37 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** + * NOTE: this API is experimental and will likely change + */ + +abstract class TermsConsumer { + + /** Starts a new term in this field; term ends with U+FFFF + * char */ + abstract DocsConsumer startTerm(char[] text, int start) throws IOException; + + /** Finishes the current term */ + abstract void finishTerm(char[] text, int start, int numDocs) throws IOException; + + /** Called when we are done adding terms to this field */ + abstract void finish() throws IOException; +} Index: src/java/org/apache/lucene/index/TermsEnum.java =================================================================== --- src/java/org/apache/lucene/index/TermsEnum.java (revision 0) +++ src/java/org/apache/lucene/index/TermsEnum.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.AttributeSource; + +/** + * NOTE: this API is experimental and will likely change + */ + +public abstract class TermsEnum extends AttributeSource { + + // nocommit -- char[] or byte[] version? + /** Seeks to the specified term. Returns true if the term + * exists. */ + public abstract boolean seek(String text) throws IOException; + + // nocommit + // abstract boolean seek(int ord) throws IOException; + + // nocommit -- String or null? + /** Increments the enumeration to the next element. True if one exists.*/ + public abstract boolean next() throws IOException; + + // nocommit -- char[] or byte[] version? + /** Returns the text for current Term in the enumeration.*/ + public abstract String text(); + + /** Returns the docFreq of the current Term in the enumeration.*/ + public abstract int docFreq(); + + /** Not all impls will implement this, eg Multi*Reader + * will not */ + public abstract long ord(); + + /** Get DocsEnum for the current term. You should not + * call {@link #next()} or {@link #seek()} until you're + * done using the DocsEnum. */ + public abstract DocsEnum docs() throws IOException; + + // nocommit -- maybe no close method? + /** Closes the enumeration to further activity, freeing resources. */ + // public abstract void close() throws IOException; +} + Index: src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- src/java/org/apache/lucene/search/TermQuery.java (revision 803321) +++ src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.IndexReader; @@ -61,12 +62,12 @@ } public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - TermDocs termDocs = reader.termDocs(term); - - if (termDocs == null) + DocsEnum docs = reader.termDocsEnum(term); + if (docs == null) { return null; + } - return new TermScorer(this, termDocs, similarity, reader.norms(term.field())); + return new TermScorer(this, docs, similarity, reader.norms(term.field())); } public Explanation explain(IndexReader reader, int doc) Index: src/java/org/apache/lucene/search/TermScorer.java =================================================================== --- src/java/org/apache/lucene/search/TermScorer.java (revision 803321) +++ src/java/org/apache/lucene/search/TermScorer.java (working copy) @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.DocsEnum; /** Expert: A Scorer for documents matching a Term. */ @@ -28,7 +28,7 @@ private static final float[] SIM_NORM_DECODER = Similarity.getNormDecoder(); private QueryWeight weight; - private TermDocs termDocs; + private DocsEnum termDocs; private byte[] norms; private float weightValue; private int doc = -1; @@ -58,7 +58,7 @@ * creates TermScorer directly, and cannot pass in a QueryWeight * object. */ - TermScorer(Weight weight, TermDocs td, Similarity similarity, byte[] norms) { + TermScorer(Weight weight, DocsEnum td, Similarity similarity, byte[] norms) { this(new QueryWeightWrapper(weight), td, similarity, norms); } @@ -75,7 +75,7 @@ * @param norms * The field norms of the document fields for the Term. */ - TermScorer(QueryWeight weight, TermDocs td, Similarity similarity, + TermScorer(QueryWeight weight, DocsEnum td, Similarity similarity, byte[] norms) { super(similarity); this.weight = weight; @@ -112,7 +112,7 @@ if (pointerMax != 0) { pointer = 0; } else { - termDocs.close(); // close stream + //termDocs.close(); // close stream doc = Integer.MAX_VALUE; // set to sentinel value return false; } @@ -153,7 +153,7 @@ if (pointerMax != 0) { pointer = 0; } else { - termDocs.close(); // close stream + //termDocs.close(); // close stream return doc = NO_MORE_DOCS; } } @@ -204,11 +204,11 @@ } // not found in cache, seek underlying stream - boolean result = termDocs.skipTo(target); - if (result) { + int newDoc = termDocs.skipTo(target); + if (newDoc != -1) { pointerMax = 1; pointer = 0; - docs[pointer] = doc = termDocs.doc(); + docs[pointer] = doc = newDoc; freqs[pointer] = termDocs.freq(); } else { doc = NO_MORE_DOCS; @@ -231,15 +231,12 @@ pointer++; } if (tf == 0) { - if (termDocs.skipTo(doc)) - { - if (termDocs.doc() == doc) - { - tf = termDocs.freq(); - } - } + int newDoc = termDocs.skipTo(doc); + if (newDoc == doc) { + tf = termDocs.freq(); + } } - termDocs.close(); + //termDocs.close(); tfExplanation.setValue(getSimilarity().tf(tf)); tfExplanation.setDescription("tf(termFreq("+query.getTerm()+")="+tf+")"); Index: src/java/org/apache/lucene/util/ArrayUtil.java =================================================================== --- src/java/org/apache/lucene/util/ArrayUtil.java (revision 803321) +++ src/java/org/apache/lucene/util/ArrayUtil.java (working copy) @@ -201,6 +201,29 @@ return grow(array, 1 + array.length); } + public static char[] shrink(char[] array, int targetSize) { + final int newSize = getShrinkSize(array.length, targetSize); + if (newSize != array.length) { + char[] newArray = new char[newSize]; + System.arraycopy(array, 0, newArray, 0, newSize); + return newArray; + } else + return array; + } + + public static char[] grow(char[] array, int minSize) { + if (array.length < minSize) { + char[] newArray = new char[getNextSize(minSize)]; + System.arraycopy(array, 0, newArray, 0, array.length); + return newArray; + } else + return array; + } + + public static char[] grow(char[] array) { + return grow(array, 1 + array.length); + } + public static byte[] shrink(byte[] array, int targetSize) { final int newSize = getShrinkSize(array.length, targetSize); if (newSize != array.length) { @@ -211,6 +234,7 @@ return array; } + /** * Returns hash of chars in range start (inclusive) to * end (inclusive) Index: src/java/org/apache/lucene/util/UnicodeUtil.java =================================================================== --- src/java/org/apache/lucene/util/UnicodeUtil.java (revision 803321) +++ src/java/org/apache/lucene/util/UnicodeUtil.java (working copy) @@ -77,11 +77,8 @@ public int length; public void setLength(int newLength) { - if (result.length < newLength) { - byte[] newArray = new byte[(int) (1.5*newLength)]; - System.arraycopy(result, 0, newArray, 0, length); - result = newArray; - } + if (result.length < newLength) + result = ArrayUtil.grow(result, newLength); length = newLength; } } @@ -92,11 +89,8 @@ public int length; public void setLength(int newLength) { - if (result.length < newLength) { - char[] newArray = new char[(int) (1.5*newLength)]; - System.arraycopy(result, 0, newArray, 0, length); - result = newArray; - } + if (result.length < newLength) + result = ArrayUtil.grow(result, newLength); length = newLength; } @@ -104,6 +98,13 @@ setLength(other.length); System.arraycopy(other.result, 0, result, 0, length); } + + public void copyText(String other) { + final int otherLength = other.length(); + setLength(otherLength); + other.getChars(0, otherLength, result, 0); + length = otherLength; + } } /** Encode characters from a char[] source, starting at Index: src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java =================================================================== --- src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (revision 803321) +++ src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (working copy) @@ -125,12 +125,13 @@ public void testWithPendingDeletes() throws IOException { // main directory - Directory dir = new RAMDirectory(); + Directory dir = new MockRAMDirectory(); // auxiliary directory - Directory aux = new RAMDirectory(); + Directory aux = new MockRAMDirectory(); setUpDirs(dir, aux); IndexWriter writer = newWriter(dir, false); + writer.addIndexesNoOptimize(new Directory[] {aux}); // Adds 10 docs, then replaces them with another 10 Index: src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 803321) +++ src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -58,11 +58,11 @@ // oldNames array. /* - public void testCreatePreLocklessCFS() throws IOException { + public void xxxtestCreatePreLocklessCFS() throws IOException { createIndex("index.cfs", true); } - public void testCreatePreLocklessNoCFS() throws IOException { + public void xxxtestCreatePreLocklessNoCFS() throws IOException { createIndex("index.nocfs", false); } */ @@ -103,13 +103,13 @@ zipFile.close(); } - public void testCreateCFS() throws IOException { + public void xxxtestCreateCFS() throws IOException { String dirName = "testindex.cfs"; createIndex(dirName, true); rmDir(dirName); } - public void testCreateNoCFS() throws IOException { + public void xxxtestCreateNoCFS() throws IOException { String dirName = "testindex.nocfs"; createIndex(dirName, true); rmDir(dirName); @@ -129,7 +129,7 @@ "24.nocfs", }; - public void testOptimizeOldIndex() throws IOException { + public void xxxtestOptimizeOldIndex() throws IOException { for(int i=0;i 1 level skipping +// - test all combinations of payloads/not and omitTF/not +// - test w/ different indexDivisor +// - test field where payload length rarely changes +// - 0-term fields +// - seek/skip to same term/doc i'm already on +// - mix in deleted docs +// - seek, skip beyond end -- assert returns false +// - seek, skip to things that don't exist -- ensure it +// goes to 1 before next one known to exist +// - skipTo(term) +// - skipTo(doc) + +public class TestFormatPostings extends LuceneTestCase { + + private static final Random RANDOM = new Random(42); + private static String[] fieldNames = new String[] {"one", "two", "three", "four"}; + + private final static int NUM_TEST_ITER = 4000; + private final static int NUM_TEST_THREADS = 3; + private final static int NUM_FIELDS = 4; + private final static int NUM_TERMS_RAND = 50; // must be > 16 to test skipping + private final static int DOC_FREQ_RAND = 500; // must be > 16 to test skipping + private final static int TERM_DOC_FREQ_RAND = 20; + + // start is inclusive and end is exclusive + public int nextInt(int start, int end) { + return start + RANDOM.nextInt(end-start); + } + + private int nextInt(int lim) { + return RANDOM.nextInt(lim); + } + + private boolean nextBoolean() { + return 0 == nextInt(1); + } + + char[] getRandomText() { + + final int len = 1+nextInt(10); + char[] buffer = new char[len+1]; + for(int i=0;i=0;i--) { + if (PostingsCodec.DEBUG) + System.out.println(" TEST: term=" + field.terms[i].text2 + " has docFreq=" + field.terms[i].docs.length); + assertTrue(termsEnum.seek(field.terms[i].text2)); + assertEquals(field.terms[i].docs.length, termsEnum.docFreq()); + } + + // Seek to non-existent empty-string term + assertFalse(termsEnum.seek("")); + + // Make sure we're now pointing to first term + assertEquals(termsEnum.text(), field.terms[0].text2); + + // Test docs enum + if (PostingsCodec.DEBUG) + System.out.println("\nTEST: docs/positions"); + termsEnum.seek(""); + upto = 0; + do { + if (nextInt(3) == 1) { + term = field.terms[upto]; + if (PostingsCodec.DEBUG) + System.out.println("TEST [" + getDesc(field, term) + "]: iterate docs..."); + DocsEnum docs = termsEnum.docs(); + int upto2 = -1; + while(upto2 < term.docs.length-1) { + // Maybe skip: + final int left = term.docs.length-upto2; + int doc; + if (nextInt(3) == 1 && left >= 1) { + int inc = 1+nextInt(left-1); + upto2 += inc; + if (PostingsCodec.DEBUG) + System.out.println("TEST [" + getDesc(field, term) + "]: skip: " + left + " docs left; skip to doc=" + term.docs[upto2] + " [" + upto2 + " of " + term.docs.length + "]"); + doc = docs.skipTo(term.docs[upto2]); + // nocommit -- test skipping to non-existent doc + assertEquals(term.docs[upto2], doc); + } else { + doc = docs.next(); + assertTrue(doc != -1); + if (PostingsCodec.DEBUG) + System.out.println("TEST [" + getDesc(field, term) + "]: got next doc..."); + upto2++; + } + assertEquals(term.docs[upto2], doc); + if (!field.omitTF) { + assertEquals(term.positions[upto2].length, docs.freq()); + if (nextInt(2) == 1) { + if (PostingsCodec.DEBUG) + System.out.println("TEST [" + getDesc(field, term, term.docs[upto2]) + "]: check positions for doc " + term.docs[upto2] + "..."); + verifyPositions(term.positions[upto2], docs.positions()); + } else if (PostingsCodec.DEBUG) + System.out.println("TEST: skip positions..."); + } else if (PostingsCodec.DEBUG) + System.out.println("TEST: skip positions: omitTF=true"); + } + + assertEquals(-1, docs.next()); + + } else if (PostingsCodec.DEBUG) + System.out.println("TEST [" + getDesc(field, term) + "]: skip docs"); + upto++; + + } while (termsEnum.next()); + + assertEquals(upto, field.terms.length); + + //termsEnum.close(); + } + } + } + + private void write(FieldInfos fieldInfos, Directory dir, FieldData[] fields) throws Throwable { + + // nocommit -- randomize this: + final int termIndexInterval = 16; + + SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, + PostingsCodecs.getDefault()); + + final FieldsConsumer consumer = state.codec.fieldsConsumer(state); + Arrays.sort(fields); + for(int i=0;i= 0); } Index: src/test/org/apache/lucene/index/TestSegmentTermDocs.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentTermDocs.java (revision 803321) +++ src/test/org/apache/lucene/index/TestSegmentTermDocs.java (working copy) @@ -55,14 +55,13 @@ SegmentReader reader = SegmentReader.get(true, info, indexDivisor); assertTrue(reader != null); assertEquals(indexDivisor, reader.getTermInfosIndexDivisor()); - SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); - assertTrue(segTermDocs != null); - segTermDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field")); - if (segTermDocs.next() == true) - { - int docId = segTermDocs.doc(); + TermDocs termDocs = reader.termDocs(); + assertTrue(termDocs != null); + termDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field")); + if (termDocs.next() == true) { + int docId = termDocs.doc(); assertTrue(docId == 0); - int freq = segTermDocs.freq(); + int freq = termDocs.freq(); assertTrue(freq == 3); } reader.close(); @@ -77,20 +76,20 @@ //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.get(true, info, indexDivisor); assertTrue(reader != null); - SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); - assertTrue(segTermDocs != null); - segTermDocs.seek(new Term("textField2", "bad")); - assertTrue(segTermDocs.next() == false); + TermDocs termDocs = reader.termDocs(); + assertTrue(termDocs != null); + termDocs.seek(new Term("textField2", "bad")); + assertTrue(termDocs.next() == false); reader.close(); } { //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.get(true, info, indexDivisor); assertTrue(reader != null); - SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); - assertTrue(segTermDocs != null); - segTermDocs.seek(new Term("junk", "bad")); - assertTrue(segTermDocs.next() == false); + TermDocs termDocs = reader.termDocs(); + assertTrue(termDocs != null); + termDocs.seek(new Term("junk", "bad")); + assertTrue(termDocs.next() == false); reader.close(); } } Index: src/test/org/apache/lucene/index/TestSegmentTermEnum.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentTermEnum.java (revision 803321) +++ src/test/org/apache/lucene/index/TestSegmentTermEnum.java (working copy) @@ -61,23 +61,6 @@ verifyDocFreq(); } - public void testPrevTermAtEnd() throws IOException - { - Directory dir = new MockRAMDirectory(); - IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); - addDoc(writer, "aaa bbb"); - writer.close(); - SegmentReader reader = SegmentReader.getOnlySegmentReader(dir); - SegmentTermEnum termEnum = (SegmentTermEnum) reader.terms(); - assertTrue(termEnum.next()); - assertEquals("aaa", termEnum.term().text()); - assertTrue(termEnum.next()); - assertEquals("aaa", termEnum.prev().text()); - assertEquals("bbb", termEnum.term().text()); - assertFalse(termEnum.next()); - assertEquals("bbb", termEnum.prev().text()); - } - private void verifyDocFreq() throws IOException { Index: src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 803321) +++ src/test/org/apache/lucene/index/TestStressIndexing2.java (working copy) @@ -73,6 +73,7 @@ // dir1 = FSDirectory.open("foofoofoo"); Directory dir2 = new MockRAMDirectory(); // mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1); + Map docs = indexRandom(10, 100, 100, dir1); indexSerial(docs, dir2); @@ -98,8 +99,12 @@ int range=r.nextInt(20)+1; Directory dir1 = new MockRAMDirectory(); Directory dir2 = new MockRAMDirectory(); + //System.out.println("iter=" + iter + " range=" + range); + //System.out.println("TEST: index random"); Map docs = indexRandom(nThreads, iter, range, dir1); + //System.out.println("TEST: index serial"); indexSerial(docs, dir2); + //System.out.println("TEST: verify"); verifyEquals(dir1, dir2, "id"); } } @@ -176,6 +181,7 @@ public Map indexRandom(int nThreads, int iterations, int range, Directory dir) throws IOException, InterruptedException { Map docs = new HashMap(); for(int iter=0;iter<3;iter++) { + //System.out.println("TEST: iter=" + iter); IndexWriter w = new MockIndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true); w.setUseCompoundFile(false); @@ -201,7 +207,8 @@ threads[i].join(); } - // w.optimize(); + // nocommit -- comment out again + //w.optimize(); w.close(); for (int i=0; i 0) { // RuntimeException instead of IOException because // super() does not throw IOException currently: - throw new RuntimeException("MockRAMDirectory: cannot close: there are still open files: " + openFiles); + Iterator it = openFiles.values().iterator(); + System.out.println("\nMockRAMDirectory open files:"); + while(it.hasNext()) { + OpenFile openFile = (OpenFile) it.next(); + System.out.println("\nfile " + openFile.name + " opened from:\n"); + openFile.stack.printStackTrace(System.out); + } + throw new RuntimeException("MockRAMDirectory: cannot close: there are still open files"); } } Index: src/test/org/apache/lucene/store/MockRAMInputStream.java =================================================================== --- src/test/org/apache/lucene/store/MockRAMInputStream.java (revision 803321) +++ src/test/org/apache/lucene/store/MockRAMInputStream.java (working copy) @@ -44,16 +44,8 @@ // all clones get closed: if (!isClone) { synchronized(dir) { - Integer v = (Integer) dir.openFiles.get(name); - // Could be null when MockRAMDirectory.crash() was called - if (v != null) { - if (v.intValue() == 1) { - dir.openFiles.remove(name); - } else { - v = new Integer(v.intValue()-1); - dir.openFiles.put(name, v); - } - } + assert dir.openFiles.containsKey(this): "input=" + name + " is not open"; + dir.openFiles.remove(this); } } } Index: src/test/org/apache/lucene/TestSearchForDuplicates.java =================================================================== --- src/test/org/apache/lucene/TestSearchForDuplicates.java (revision 803321) +++ src/test/org/apache/lucene/TestSearchForDuplicates.java (working copy) @@ -94,6 +94,9 @@ for (int j = 0; j < MAX_DOCS; j++) { Document d = new Document(); d.add(new Field(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES, Field.Index.ANALYZED)); + + // NOTE: this ID_FIELD produces no tokens since + // SimpleAnalyzer discards numbers d.add(new Field(ID_FIELD, Integer.toString(j), Field.Store.YES, Field.Index.ANALYZED)); writer.addDocument(d); }