Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java =================================================================== --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java (revision 1051521) +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java (working copy) @@ -19,9 +19,13 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.codecs.PrefixCodedTermState; + +import java.io.IOException; import java.util.Arrays; import java.util.Comparator; @@ -91,20 +95,11 @@ } @Override - public void cacheCurrentTerm() { - } - - @Override public BytesRef term() { return br; } @Override - public long ord() { - return upto - start; - } - - @Override public int docFreq() { return terms[upto].getAssociatedDocuments().length; } @@ -129,5 +124,19 @@ public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } + + @Override + public TermState termState() throws IOException { + final PrefixCodedTermState state = new PrefixCodedTermState(); + state.docFreq = docFreq(); + state.ord = upto - start; + return state; + } + + @Override + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + assert state != null && state instanceof PrefixCodedTermState; + return seek(((PrefixCodedTermState)state).ord()); // just use the ord for simplicity + } } Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1051521) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -38,6 +38,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Fields; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.FieldsEnum; @@ -48,6 +49,8 @@ import org.apache.lucene.index.TermPositionVector; import org.apache.lucene.index.TermVectorMapper; import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -877,15 +880,6 @@ } @Override - public void cacheCurrentTerm() { - } - - @Override - public long ord() { - return termUpto; - } - - @Override public int docFreq() { return 1; } @@ -910,6 +904,20 @@ public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } + + @Override + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + assert state != null && state instanceof PrefixCodedTermState; + return this.seek(((PrefixCodedTermState)state).ord); + } + + @Override + public TermState termState() throws IOException { + PrefixCodedTermState termState = new PrefixCodedTermState(); + termState.docFreq = 1; + termState.ord = termUpto; + return termState; + } } private class MemoryDocsEnum extends DocsEnum { Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -27,6 +27,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -637,7 +638,12 @@ if (hasOrd) { long ord = -1; try { - ord = terms.ord(); + TermState termState = terms.termState(); + if (termState instanceof PrefixCodedTermState) { + ord = ((PrefixCodedTermState) termState).ord(); + } else { + hasOrd = false; + } } catch (UnsupportedOperationException uoe) { hasOrd = false; } Index: lucene/src/java/org/apache/lucene/index/FilterIndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -130,11 +130,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - in.cacheCurrentTerm(); - } - - @Override public SeekStatus seek(long ord) throws IOException { return in.seek(ord); } @@ -150,11 +145,6 @@ } @Override - public long ord() throws IOException { - return in.ord(); - } - - @Override public int docFreq() { return in.docFreq(); } @@ -173,6 +163,16 @@ public Comparator getComparator() throws IOException { return in.getComparator(); } + + @Override + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + return in.seek(term, state); + } + + @Override + public TermState termState() throws IOException { + return in.termState(); + } } /** Base class for filtering {@link DocsEnum} implementations. */ Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1070,6 +1070,49 @@ return null; } } + + /** + * Returns {@link DocsEnum} for the specified field, term and + * {@link TermState}. This may return null, if either the field or the term + * does not exists or the {@link TermState} is invalid for the underlying + * implementation.*/ + public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException { + assert state != null; + assert field != null; + assert term != null; + final Fields fields = fields(); + if (fields == null) { + return null; + } + final Terms terms = fields.terms(field); + if (terms != null) { + return terms.docs(skipDocs, term, state, null); + } else { + return null; + } + } + + /** + * Returns {@link DocsAndPositionsEnum} for the specified field, term and + * {@link TermState}. This may return null, if either the field or the term + * does not exists, the {@link TermState} is invalid for the underlying + * implementation, or positions were not stored for this term.*/ + public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException { + assert state != null; + assert field != null; + assert term != null; + final Fields fields = fields(); + if (fields == null) { + return null; + } + final Terms terms = fields.terms(field); + if (terms != null) { + return terms.docsAndPositions(skipDocs, term, state, null); + } else { + return null; + } + } + /** Deletes the document numbered docNum. Once a document is * deleted it will not appear in TermDocs or TermPositions enumerations. @@ -1126,7 +1169,7 @@ if (docs == null) return 0; int n = 0; int doc; - while ((doc = docs.nextDoc()) != docs.NO_MORE_DOCS) { + while ((doc = docs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { deleteDocument(doc); n++; } Index: lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (working copy) @@ -91,13 +91,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - for(int i=0;i getComparator() { return termComp; } @@ -215,11 +208,6 @@ throw new UnsupportedOperationException(); } - @Override - public long ord() throws IOException { - throw new UnsupportedOperationException(); - } - private void pullTop() { // extract all subs from the queue that have the same // top term @@ -434,4 +422,24 @@ } } } + + @Override + public SeekStatus seek(BytesRef term, TermState state /*ignored*/) throws IOException { + assert term != null; + return seek(term); + } + + @Override + public TermState termState() throws IOException { + return new TermState() { + private final int docFreq = MultiTermsEnum.this.docFreq(); + @Override + public int docFreq() { + return docFreq; + } + @Override + public void copy(TermState other) { + } + }; + } } Index: lucene/src/java/org/apache/lucene/index/TermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermState.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/TermState.java (revision 0) @@ -0,0 +1,54 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Holds all state required for {@link TermsEnum} to produce a {@link DocsEnum} + * without re-seeking the terms dict. + * + * @lucene.experimental + */ +public abstract class TermState implements Cloneable { + + /** + * Copies the content of the given {@link TermState} to this instance + * + * @param other + * the TermState to copy + */ + public abstract void copy(TermState other); + + /** + * Returns the document frequency of the term this {@link TermState} was + * created for in the creating {@link TermsEnum} + * @return the document frequency of the term this {@link TermState} was + * created for in the creating {@link TermsEnum} + */ + public abstract int docFreq(); + + @Override + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException cnse) { + // should not happen + throw new RuntimeException(cnse); + } + } + +} \ No newline at end of file Index: lucene/src/java/org/apache/lucene/index/Terms.java =================================================================== --- lucene/src/java/org/apache/lucene/index/Terms.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/Terms.java (working copy) @@ -80,6 +80,39 @@ } } + /** + * Expert: Get {@link DocsEnum} for the specified term and {@link TermState}. + * This method may return null if the term does not exist. + * + * @see TermsEnum#termState() + * @see TermsEnum#seek(BytesRef, TermState) */ + public DocsEnum docs(Bits skipDocs, BytesRef text, TermState termState, + DocsEnum reuse) throws IOException { + final TermsEnum termsEnum = getThreadTermsEnum(); + if (termsEnum.seek(text, termState) == TermsEnum.SeekStatus.FOUND) { + return termsEnum.docs(skipDocs, reuse); + } else { + return null; + } + } + + /** + * Get {@link DocsEnum} for the specified term and {@link TermState}. This + * method will may return null if the term does not exists, or positions were + * not indexed. + * + * @see TermsEnum#termState() + * @see TermsEnum#seek(BytesRef, TermState) */ + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef text, + TermState termState, DocsAndPositionsEnum reuse) throws IOException { + final TermsEnum termsEnum = getThreadTermsEnum(); + if (termsEnum.seek(text, termState) == TermsEnum.SeekStatus.FOUND) { + return termsEnum.docsAndPositions(skipDocs, reuse); + } else { + return null; + } + } + public long getUniqueTermCount() throws IOException { throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()"); } Index: lucene/src/java/org/apache/lucene/index/TermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermsEnum.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/TermsEnum.java (working copy) @@ -73,6 +73,20 @@ * may be before or after the current ord. See {@link * #seek(BytesRef)}. */ public abstract SeekStatus seek(long ord) throws IOException; + + /** + * Expert: Seeks to the specified term by {@link TermState} as previously + * returned by {@link #termState()}. The given {@link BytesRef} must hold the + * same text as used for positioning the TermsEnum before {@link #termState()} + * was called. Callers should maintain the {@link BytesRef} obtained from + * {@link #term()} together with the {@link TermState} to use this method. + * Low-level implementations will position the TermsEnum without re-seeking + * the term dictionary. Using this method with a {@link TermState} obtained + * from a different {@link Terms} instance will leave the {@link TermsEnum} in + * undefined state. + * */ + public abstract SeekStatus seek(BytesRef text, TermState state) throws IOException; + /** Increments the enumeration to the next element. * Returns the resulting term, or null if the end was @@ -85,20 +99,12 @@ * or after seek returns {@link SeekStatus#END}.*/ public abstract BytesRef term() throws IOException; - /** Returns ordinal position for current term. This is an - * optional method (the codec may throw {@link - * UnsupportedOperationException}). Do not call this - * before calling {@link #next} for the first time or after - * {@link #next} returns null or {@link #seek} returns - * END; */ - public abstract long ord() throws IOException; - /** Returns the number of documents containing the current * term. Do not call this before calling next() for the * first time, after next() returns null or seek returns * {@link SeekStatus#END}.*/ public abstract int docFreq(); - + /** Get {@link DocsEnum} for the current term. Do not * call this before calling {@link #next} or {@link * #seek} for the first time. This method will not @@ -116,6 +122,15 @@ * the postings by this codec. */ public abstract DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; + /** + * Expert: Returns the TermsEnums internal state to position the TermsEnum + * without re-seeking the term dictionary. + * + * @see TermState + * @see #seek(BytesRef, TermState) + */ + public abstract TermState termState() throws IOException; + /** Return the {@link BytesRef} Comparator used to sort * terms provided by the iterator. This may return * null if there are no terms. Callers may invoke this @@ -123,10 +138,6 @@ * instance & reuse it. */ public abstract Comparator getComparator() throws IOException; - /** Optional optimization hint: informs the codec that the - * current term is likely to be re-seek'd-to soon. */ - public abstract void cacheCurrentTerm() throws IOException; - /** An empty TermsEnum for quickly returning an empty instance e.g. * in {@link org.apache.lucene.search.MultiTermQuery} *

Please note: This enum should be unmodifiable, @@ -142,9 +153,6 @@ public SeekStatus seek(long ord) { return SeekStatus.END; } @Override - public void cacheCurrentTerm() {} - - @Override public BytesRef term() { throw new IllegalStateException("this method should never be called"); } @@ -160,11 +168,6 @@ } @Override - public long ord() { - throw new IllegalStateException("this method should never be called"); - } - - @Override public DocsEnum docs(Bits bits, DocsEnum reuse) { throw new IllegalStateException("this method should never be called"); } @@ -183,5 +186,15 @@ public synchronized AttributeSource attributes() { return super.attributes(); } + + @Override + public TermState termState() throws IOException { + throw new IllegalStateException("this method should never be called"); + } + + @Override + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + throw new IllegalStateException("this method should never be called"); + } }; } Index: lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (working copy) @@ -42,17 +42,17 @@ public abstract void init(IndexInput termsIn) throws IOException; /** Return a newly created empty TermState */ - public abstract TermState newTermState() throws IOException; + public abstract PrefixCodedTermState newTermState() throws IOException; - public abstract void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState state, boolean isIndexTerm) throws IOException; + public abstract void readTerm(IndexInput termsIn, FieldInfo fieldInfo, PrefixCodedTermState state, boolean isIndexTerm) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsEnum docs(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsEnum reuse) throws IOException; + public abstract DocsEnum docs(FieldInfo fieldInfo, PrefixCodedTermState state, Bits skipDocs, DocsEnum reuse) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; + public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, PrefixCodedTermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; public abstract void close() throws IOException; } Index: lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermState.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermState.java (revision 0) @@ -0,0 +1,54 @@ +package org.apache.lucene.index.codecs; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.TermState; + +/** + * Holds all state required for {@link PostingsReaderBase} + * to produce a {@link DocsEnum} without re-seeking the + * terms dict. + */ +public class PrefixCodedTermState extends TermState { + public int docFreq; // how many docs have this term + public long ord; // ord for this term + public long filePointer; // fp into the terms dict primary file (_X.tis) + + @Override + public void copy(TermState _other) { + PrefixCodedTermState other = (PrefixCodedTermState) _other; + ord = other.ord; + filePointer = other.filePointer; + docFreq = other.docFreq; + } + + @Override + public String toString() { + return super.toString() + "[ord=" + ord + ", tis.filePointer=" + filePointer + + "]"; + } + + public long ord() { + return ord; + } + + @Override + public int docFreq() { + return docFreq; + } + +} Index: lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.Directory; @@ -68,7 +69,7 @@ private final Comparator termComp; // Caches the most recently looked-up field + terms: - private final DoubleBarrelLRUCache termsCache; + private final DoubleBarrelLRUCache termsCache; // Reads the terms index private TermsIndexReaderBase indexReader; @@ -84,11 +85,6 @@ public FieldAndTerm() { } - public FieldAndTerm(String field, BytesRef term) { - this.field = field; - this.term = new BytesRef(term); - } - public FieldAndTerm(FieldAndTerm other) { field = other.field; term = new BytesRef(other.term); @@ -116,7 +112,7 @@ throws IOException { this.postingsReader = postingsReader; - termsCache = new DoubleBarrelLRUCache(termsCacheSize); + termsCache = new DoubleBarrelLRUCache(termsCacheSize); this.termComp = termComp; @@ -282,10 +278,10 @@ } // Iterates through terms in this field - private class SegmentTermsEnum extends TermsEnum { + private final class SegmentTermsEnum extends TermsEnum { private final IndexInput in; private final DeltaBytesReader bytesReader; - private final TermState state; + private final PrefixCodedTermState state; private boolean seekPending; private final TermsIndexReaderBase.TermsIndexResult indexResult = new TermsIndexReaderBase.TermsIndexResult(); private final FieldAndTerm fieldTerm = new FieldAndTerm(); @@ -304,33 +300,20 @@ return termComp; } - @Override - public void cacheCurrentTerm() { - TermState stateCopy = (TermState) state.clone(); - stateCopy.filePointer = in.getFilePointer(); - termsCache.put(new FieldAndTerm(fieldInfo.name, bytesReader.term), - stateCopy); - } - /** Seeks until the first term that's >= the provided * text; returns SeekStatus.FOUND if the exact term * is found, SeekStatus.NOT_FOUND if a different term * was found, SeekStatus.END if we hit EOF */ @Override - public SeekStatus seek(BytesRef term, boolean useCache) throws IOException { + public SeekStatus seek(final BytesRef term, final boolean useCache) throws IOException { // Check cache fieldTerm.term = term; - TermState cachedState; if (useCache) { - cachedState = termsCache.get(fieldTerm); + final PrefixCodedTermState cachedState = termsCache.get(fieldTerm); if (cachedState != null) { - state.copy(cachedState); - seekPending = true; - bytesReader.term.copy(term); + setTermState(term, cachedState); return SeekStatus.FOUND; } - } else { - cachedState = null; } boolean doSeek = true; @@ -387,16 +370,9 @@ while(next() != null) { final int cmp = termComp.compare(bytesReader.term, term); if (cmp == 0) { - if (doSeek && useCache) { - // Store in cache - FieldAndTerm entryKey = new FieldAndTerm(fieldTerm); - cachedState = (TermState) state.clone(); - // this is fp after current term - cachedState.filePointer = in.getFilePointer(); - termsCache.put(entryKey, cachedState); + cacheTerm(fieldTerm); } - return SeekStatus.FOUND; } else if (cmp > 0) { return SeekStatus.NOT_FOUND; @@ -412,6 +388,24 @@ return SeekStatus.END; } + private final void setTermState(final BytesRef term, final TermState termState) { + assert termState != null; + assert term != null; + state.copy(termState); + seekPending = true; + bytesReader.term.copy(term); + } + + private final void cacheTerm(FieldAndTerm other) { + // Store in cache + final FieldAndTerm entryKey = new FieldAndTerm(other); + final PrefixCodedTermState cachedState = (PrefixCodedTermState) state.clone(); + // this is fp after current term + cachedState.filePointer = in.getFilePointer(); + termsCache.put(entryKey, cachedState); + } + + @Override public SeekStatus seek(long ord) throws IOException { @@ -452,11 +446,6 @@ } @Override - public long ord() { - return state.ord; - } - - @Override public BytesRef next() throws IOException { if (seekPending) { @@ -494,7 +483,7 @@ @Override public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { - DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, skipDocs, reuse); + final DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, skipDocs, reuse); assert docsEnum != null; return docsEnum; } @@ -507,6 +496,22 @@ return postingsReader.docsAndPositions(fieldInfo, state, skipDocs, reuse); } } + + @Override + public SeekStatus seek(BytesRef term, TermState otherState) throws IOException { + assert otherState != null && otherState instanceof PrefixCodedTermState; + assert otherState.getClass() == this.state.getClass(); + assert ((PrefixCodedTermState)otherState).ord() < numTerms; + setTermState(term, otherState); + return SeekStatus.FOUND; + } + + @Override + public TermState termState() throws IOException { + final PrefixCodedTermState newTermState = (PrefixCodedTermState) state.clone(); + newTermState.filePointer = in.getFilePointer(); + return newTermState; + } } } } Index: lucene/src/java/org/apache/lucene/index/codecs/TermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/TermState.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/codecs/TermState.java (working copy) @@ -1,56 +0,0 @@ -package org.apache.lucene.index.codecs; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.DocsEnum; // for javadocs - -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs - -/** - * Holds all state required for {@link StandardPostingsReader} - * to produce a {@link DocsEnum} without re-seeking the - * terms dict. - * @lucene.experimental - */ - -public class TermState implements Cloneable { - public long ord; // ord for this term - public long filePointer; // fp into the terms dict primary file (_X.tis) - public int docFreq; // how many docs have this term - - public void copy(TermState other) { - ord = other.ord; - filePointer = other.filePointer; - docFreq = other.docFreq; - } - - @Override - public Object clone() { - try { - return super.clone(); - } catch (CloneNotSupportedException cnse) { - // should not happen - throw new RuntimeException(cnse); - } - } - - @Override - public String toString() { - return "tis.fp=" + filePointer + " docFreq=" + docFreq + " ord=" + ord; - } -} Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (working copy) @@ -33,6 +33,7 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.CompoundFileReader; @@ -742,21 +743,11 @@ } @Override - public void cacheCurrentTerm() throws IOException { - getTermsDict().cacheCurrentTerm(termEnum); - } - - @Override public SeekStatus seek(long ord) throws IOException { throw new UnsupportedOperationException(); } @Override - public long ord() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override public SeekStatus seek(BytesRef term, boolean useCache) throws IOException { if (DEBUG_SURROGATES) { System.out.println("TE.seek target=" + UnicodeUtil.toHexString(term.utf8ToString())); @@ -971,6 +962,26 @@ } return docsPosEnum.reset(termEnum, skipDocs); } + + @Override + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + assert term != null; + return seek(term); + } + + @Override + public TermState termState() throws IOException { + return new TermState() { + private final int docFreq = termEnum.docFreq(); + @Override + public int docFreq() { + return docFreq; + } + @Override + public void copy(TermState other) { + } + }; + } } private final class PreDocsEnum extends DocsEnum { Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (working copy) @@ -22,8 +22,9 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.codecs.TermState; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl.Document; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl.Position; import org.apache.lucene.store.IndexInput; @@ -57,14 +58,14 @@ wrappedPostingsReader.init(termsIn); } - private static class PulsingTermState extends TermState { + private static final class PulsingTermState extends PrefixCodedTermState { private Document docs[]; - private TermState wrappedTermState; + private PrefixCodedTermState wrappedTermState; private boolean pendingIndexTerm; + @Override public Object clone() { - PulsingTermState clone; - clone = (PulsingTermState) super.clone(); + final PulsingTermState clone = (PulsingTermState) super.clone(); clone.docs = docs.clone(); for(int i=0;i> result = fstEnum.next(); @@ -176,11 +173,6 @@ } @Override - public long ord() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override public SeekStatus seek(long ord) { throw new UnsupportedOperationException(); } @@ -215,6 +207,25 @@ } return docsAndPositionsEnum.reset(docsStart, skipDocs); } + + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + assert term != null; + return seek(term); + } + + @Override + public TermState termState() throws IOException { + return new TermState() { + private final int docFreq =SimpleTextTermsEnum.this.docFreq; + @Override + public int docFreq() { + return docFreq; + } + @Override + public void copy(TermState other) { + } + }; + } @Override public Comparator getComparator() { @@ -440,7 +451,6 @@ } private class SimpleTextTerms extends Terms { - private final String field; private final long termsStart; private final boolean omitTF; private FST> fst; @@ -448,7 +458,6 @@ private final BytesRef scratch = new BytesRef(10); public SimpleTextTerms(String field, long termsStart) throws IOException { - this.field = StringHelper.intern(field); this.termsStart = termsStart; omitTF = fieldInfos.fieldInfo(field).omitTermFreqAndPositions; loadTerms(); Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (working copy) @@ -26,8 +26,9 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.TermState; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -83,20 +84,22 @@ } // Must keep final because we do non-standard clone - private final static class DocTermState extends TermState { + private final static class StandardTermState extends PrefixCodedTermState { long freqOffset; long proxOffset; int skipOffset; - + + @Override public Object clone() { - DocTermState other = new DocTermState(); + StandardTermState other = new StandardTermState(); other.copy(this); return other; } + @Override public void copy(TermState _other) { super.copy(_other); - DocTermState other = (DocTermState) _other; + StandardTermState other = (StandardTermState) _other; freqOffset = other.freqOffset; proxOffset = other.proxOffset; skipOffset = other.skipOffset; @@ -108,8 +111,8 @@ } @Override - public TermState newTermState() { - return new DocTermState(); + public PrefixCodedTermState newTermState() { + return new StandardTermState(); } @Override @@ -126,10 +129,9 @@ } @Override - public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState termState, boolean isIndexTerm) + public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, PrefixCodedTermState termState, boolean isIndexTerm) throws IOException { - - final DocTermState docTermState = (DocTermState) termState; + final StandardTermState docTermState = (StandardTermState) termState; if (isIndexTerm) { docTermState.freqOffset = termsIn.readVLong(); @@ -153,7 +155,7 @@ } @Override - public DocsEnum docs(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, PrefixCodedTermState termState, Bits skipDocs, DocsEnum reuse) throws IOException { SegmentDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SegmentDocsEnum)) { docsEnum = new SegmentDocsEnum(freqIn); @@ -166,11 +168,11 @@ docsEnum = new SegmentDocsEnum(freqIn); } } - return docsEnum.reset(fieldInfo, (DocTermState) termState, skipDocs); + return docsEnum.reset(fieldInfo, (StandardTermState) termState, skipDocs); } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, PrefixCodedTermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { if (fieldInfo.omitTermFreqAndPositions) { return null; } @@ -189,7 +191,7 @@ docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); } } - return docsEnum.reset(fieldInfo, (DocTermState) termState, skipDocs); + return docsEnum.reset(fieldInfo, (StandardTermState) termState, skipDocs); } else { SegmentDocsAndPositionsEnum docsEnum; if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsEnum)) { @@ -203,7 +205,7 @@ docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); } } - return docsEnum.reset(fieldInfo, (DocTermState) termState, skipDocs); + return docsEnum.reset(fieldInfo, (StandardTermState) termState, skipDocs); } } @@ -233,7 +235,7 @@ this.freqIn = (IndexInput) freqIn.clone(); } - public SegmentDocsEnum reset(FieldInfo fieldInfo, DocTermState termState, Bits skipDocs) throws IOException { + public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs) throws IOException { omitTF = fieldInfo.omitTermFreqAndPositions; if (omitTF) { freq = 1; @@ -407,7 +409,7 @@ this.proxIn = (IndexInput) proxIn.clone(); } - public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, DocTermState termState, Bits skipDocs) throws IOException { + public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs) throws IOException { assert !fieldInfo.omitTermFreqAndPositions; assert !fieldInfo.storePayloads; @@ -594,7 +596,7 @@ this.proxIn = (IndexInput) proxIn.clone(); } - public SegmentDocsAndPositionsAndPayloadsEnum reset(FieldInfo fieldInfo, DocTermState termState, Bits skipDocs) throws IOException { + public SegmentDocsAndPositionsAndPayloadsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs) throws IOException { assert !fieldInfo.omitTermFreqAndPositions; assert fieldInfo.storePayloads; if (payload == null) { Index: lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (working copy) @@ -21,9 +21,15 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; +import org.apache.lucene.util.PerReaderTermState; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray; class ConstantScoreAutoRewrite extends TermCollectingRewrite { @@ -71,8 +77,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/) { - topLevel.add(new TermQuery(term, docFreq), BooleanClause.Occur.SHOULD); + protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/, PerReaderTermState states) { + topLevel.add(new TermQuery(term, states), BooleanClause.Occur.SHOULD); } @Override @@ -98,9 +104,10 @@ final BytesRefHash pendingTerms = col.pendingTerms; final int sort[] = pendingTerms.sort(col.termsEnum.getComparator()); for(int i = 0; i < size; i++) { + final int pos = sort[i]; // docFreq is not used for constant score here, we pass 1 // to explicitely set a fake value, so it's not calculated - addClause(bq, placeholderTerm.createTerm(pendingTerms.get(sort[i], new BytesRef())), 1, 1.0f); + addClause(bq, placeholderTerm.createTerm(pendingTerms.get(pos, new BytesRef())), 1, 1.0f, col.array.termState[pos]); } // Strip scores final Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq)); @@ -123,12 +130,21 @@ @Override public boolean collect(BytesRef bytes) throws IOException { - pendingTerms.add(bytes); + int pos = pendingTerms.add(bytes); docVisitCount += termsEnum.docFreq(); if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) { hasCutOff = true; return false; } + + final TermState termState = termsEnum.termState(); + assert termState != null; + if (pos < 0) { + pos = (-pos)-1; + array.termState[pos].register(termState, reader); + } else { + array.termState[pos] = new PerReaderTermState(termStateReaderView, termState, reader); + } return true; } @@ -137,7 +153,8 @@ TermsEnum termsEnum; final int docCountCutoff, termCountLimit; - final BytesRefHash pendingTerms = new BytesRefHash(); + final TermStateByteStart array = new TermStateByteStart(16); + final BytesRefHash pendingTerms = new BytesRefHash(new ByteBlockPool(new ByteBlockPool.DirectAllocator()), 16, array); } @Override @@ -166,4 +183,40 @@ return true; } + + /** Special implementation of BytesStartArray that keeps parallel arrays for {@link PerReaderTermState} */ + static final class TermStateByteStart extends DirectBytesStartArray { + PerReaderTermState[] termState; + + public TermStateByteStart(int initSize) { + super(initSize); + } + + @Override + public int[] init() { + final int[] ord = super.init(); + termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + assert termState.length >= ord.length; + return ord; + } + + @Override + public int[] grow() { + final int[] ord = super.grow(); + if (termState.length < ord.length) { + PerReaderTermState[] tmpTermState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + System.arraycopy(termState, 0, tmpTermState, 0, termState.length); + termState = tmpTermState; + } + assert termState.length >= ord.length; + return ord; + } + + @Override + public int[] clear() { + termState = null; + return super.clear(); + } + + } } Index: lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java (working copy) @@ -21,6 +21,7 @@ import java.util.Comparator; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; @@ -142,11 +143,6 @@ } @Override - public long ord() throws IOException { - return tenum.ord(); - } - - @Override public DocsEnum docs(Bits bits, DocsEnum reuse) throws IOException { return tenum.docs(bits, reuse); } @@ -155,12 +151,24 @@ public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse) throws IOException { return tenum.docsAndPositions(bits, reuse); } - + + /** This enum does not support seeking! + * @throws UnsupportedOperationException + */ @Override - public void cacheCurrentTerm() throws IOException { - tenum.cacheCurrentTerm(); + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + throw new UnsupportedOperationException(getClass().getName()+" does not support seeking"); } - + + /** + * Returns the filtered enums term state + */ + @Override + public TermState termState() throws IOException { + assert tenum != null; + return tenum.termState(); + } + @SuppressWarnings("fallthrough") @Override public BytesRef next() throws IOException { Index: lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeImpl; @@ -245,11 +246,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - actualEnum.cacheCurrentTerm(); - } - - @Override public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { return actualEnum.docs(skipDocs, reuse); } @@ -260,14 +256,18 @@ return actualEnum.docsAndPositions(skipDocs, reuse); } + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + return actualEnum.seek(term, state); + } + @Override - public Comparator getComparator() throws IOException { - return actualEnum.getComparator(); + public TermState termState() throws IOException { + return actualEnum.termState(); } @Override - public long ord() throws IOException { - return actualEnum.ord(); + public Comparator getComparator() throws IOException { + return actualEnum.getComparator(); } @Override Index: lucene/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/src/java/org/apache/lucene/search/IndexSearcher.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/IndexSearcher.java (working copy) @@ -126,8 +126,9 @@ */ @Override public void close() throws IOException { - if(closeReader) + if (closeReader) { reader.close(); + } } // inherit javadoc Index: lucene/src/java/org/apache/lucene/search/MultiTermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.PerReaderTermState; /** * An abstract {@link Query} that matches documents @@ -159,8 +160,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost) { - final TermQuery tq = new TermQuery(term, docCount); + protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) { + final TermQuery tq = new TermQuery(term, states); tq.setBoost(boost); topLevel.add(tq, BooleanClause.Occur.SHOULD); } @@ -200,8 +201,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost) { - final Query q = new ConstantScoreQuery(new QueryWrapperFilter(new TermQuery(term, docFreq))); + protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) { + final Query q = new ConstantScoreQuery(new QueryWrapperFilter(new TermQuery(term, states))); q.setBoost(boost); topLevel.add(q, BooleanClause.Occur.SHOULD); } Index: lucene/src/java/org/apache/lucene/search/Query.java =================================================================== --- lucene/src/java/org/apache/lucene/search/Query.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/Query.java (working copy) @@ -135,7 +135,7 @@ BooleanClause[] clauses = null; // check if we can split the query into clauses boolean splittable = (query instanceof BooleanQuery); - if(splittable){ + if (splittable) { BooleanQuery bq = (BooleanQuery) query; splittable = bq.isCoordDisabled(); clauses = bq.getClauses(); @@ -143,7 +143,7 @@ splittable = (clauses[j].getOccur() == BooleanClause.Occur.SHOULD); } } - if(splittable){ + if (splittable) { for (int j = 0; j < clauses.length; j++) { uniques.add(clauses[j].getQuery()); } Index: lucene/src/java/org/apache/lucene/search/ScoringRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (working copy) @@ -18,24 +18,18 @@ */ import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.PriorityQueue; -import java.util.Comparator; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.MultiTermQuery.RewriteMethod; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; +import org.apache.lucene.util.PerReaderTermState; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray; @@ -62,11 +56,12 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost) { - final TermQuery tq = new TermQuery(term, docCount); + protected void addClause(BooleanQuery topLevel, Term term, int docCount, + float boost, PerReaderTermState states) { + final TermQuery tq = new TermQuery(term, states); tq.setBoost(boost); - topLevel.add(tq, BooleanClause.Occur.SHOULD); - } + topLevel.add(tq, BooleanClause.Occur.SHOULD); + } @Override protected void checkMaxClauseCount(int count) { @@ -77,7 +72,7 @@ // Make sure we are still a singleton even after deserializing protected Object readResolve() { return SCORING_BOOLEAN_QUERY_REWRITE; - } + } }; /** Like {@link #SCORING_BOOLEAN_QUERY_REWRITE} except @@ -123,13 +118,13 @@ final int size = col.terms.size(); if (size > 0) { final int sort[] = col.terms.sort(col.termsEnum.getComparator()); - final int[] docFreq = col.array.docFreq; final float[] boost = col.array.boost; + final PerReaderTermState[] termStates = col.array.termState; for (int i = 0; i < size; i++) { final int pos = sort[i]; final Term term = placeholderTerm.createTerm(col.terms.get(pos, new BytesRef())); - assert reader.docFreq(term) == docFreq[pos]; - addClause(result, term, docFreq[pos], query.getBoost() * boost[pos]); + assert reader.docFreq(term) == termStates[pos].docFreq(); + addClause(result, term, termStates[pos].docFreq(), query.getBoost() * boost[pos], termStates[pos]); } } query.incTotalNumberOfTerms(size); @@ -152,15 +147,17 @@ @Override public boolean collect(BytesRef bytes) throws IOException { final int e = terms.add(bytes); + final TermState state = termsEnum.termState(); + assert state != null; if (e < 0 ) { // duplicate term: update docFreq final int pos = (-e)-1; - array.docFreq[pos] += termsEnum.docFreq(); + array.termState[pos].register(state, reader); assert array.boost[pos] == boostAtt.getBoost() : "boost should be equal in all segment TermsEnums"; } else { // new entry: we populate the entry initially - array.docFreq[e] = termsEnum.docFreq(); array.boost[e] = boostAtt.getBoost(); + array.termState[e] = new PerReaderTermState(termStateReaderView, state, reader); ScoringRewrite.this.checkMaxClauseCount(terms.size()); } return true; @@ -169,8 +166,8 @@ /** Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq */ static final class TermFreqBoostByteStart extends DirectBytesStartArray { - int[] docFreq; float[] boost; + PerReaderTermState[] termState; public TermFreqBoostByteStart(int initSize) { super(initSize); @@ -180,24 +177,28 @@ public int[] init() { final int[] ord = super.init(); boost = new float[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_FLOAT)]; - docFreq = new int[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_INT)]; - assert boost.length >= ord.length && docFreq.length >= ord.length; + termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + assert termState.length >= ord.length && boost.length >= ord.length; return ord; } @Override public int[] grow() { final int[] ord = super.grow(); - docFreq = ArrayUtil.grow(docFreq, ord.length); boost = ArrayUtil.grow(boost, ord.length); - assert boost.length >= ord.length && docFreq.length >= ord.length; + if (termState.length < ord.length) { + PerReaderTermState[] tmpTermState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + System.arraycopy(termState, 0, tmpTermState, 0, termState.length); + termState = tmpTermState; + } + assert termState.length >= ord.length && boost.length >= ord.length; return ord; } @Override public int[] clear() { boost = null; - docFreq = null; + termState = null; return super.clear(); } Index: lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java (working copy) @@ -18,8 +18,8 @@ */ import java.io.IOException; -import java.io.Serializable; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Comparator; @@ -30,21 +30,29 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PerReaderTermState; +import org.apache.lucene.util.PerReaderTermState.ReaderView; import org.apache.lucene.util.ReaderUtil; abstract class TermCollectingRewrite extends MultiTermQuery.RewriteMethod { + /** Return a suitable top-level Query for holding all expanded terms. */ protected abstract Q getTopLevelQuery() throws IOException; /** Add a MultiTermQuery term to the top-level query */ - protected abstract void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException; + protected final void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException { + addClause(topLevel, term, docCount, boost, null); + } + + protected abstract void addClause(Q topLevel, Term term, int docCount, float boost, PerReaderTermState states) throws IOException; + protected final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException { final List subReaders = new ArrayList(); ReaderUtil.gatherSubReaders(subReaders, reader); Comparator lastTermComp = null; - + collector.subReaders(subReaders); for (IndexReader r : subReaders) { final Fields fields = r.fields(); if (fields == null) { @@ -71,9 +79,9 @@ lastTermComp = newTermComp; collector.setNextEnum(termsEnum); + collector.setNextReader(r); BytesRef bytes; while ((bytes = termsEnum.next()) != null) { - termsEnum.cacheCurrentTerm(); if (!collector.collect(bytes)) return; // interrupt whole term collection, so also don't iterate other subReaders } @@ -81,6 +89,12 @@ } protected static abstract class TermCollector { + protected IndexReader reader; + protected ReaderView termStateReaderView; + + public void subReaders(Collection readers) { + termStateReaderView = new ReaderView(readers); + } /** attributes used for communication with the enum */ public final AttributeSource attributes = new AttributeSource(); @@ -89,5 +103,9 @@ /** the next segment's {@link TermsEnum} that is used to collect terms */ public abstract void setNextEnum(TermsEnum termsEnum) throws IOException; + + public void setNextReader(IndexReader reader) { + this.reader = reader; + } } } Index: lucene/src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermQuery.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -23,7 +23,11 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; import org.apache.lucene.search.Explanation.IDFExplanation; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PerReaderTermState; import org.apache.lucene.util.ToStringUtils; /** A Query that matches documents containing a term. @@ -31,15 +35,16 @@ */ public class TermQuery extends Query { private final Term term; - private final int docFreq; + private int docFreq; + private transient PerReaderTermState perReaderTermState; private class TermWeight extends Weight { private final Similarity similarity; private float value; - private float idf; + private final float idf; private float queryNorm; private float queryWeight; - private IDFExplanation idfExp; + private final IDFExplanation idfExp; public TermWeight(Searcher searcher) throws IOException { @@ -76,17 +81,34 @@ @Override public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), - term.field(), - term.bytes()); - - if (docs == null) { - return null; + final String field = term.field(); + final BytesRef bytes = term.bytes(); + final DocsEnum docs = getDocsEnum(reader, field, bytes); + return (docs == null) ? null : new TermScorer(this, docs, similarity, reader.norms(field)); + } + + private DocsEnum getDocsEnum(IndexReader reader, String field, BytesRef bytes) throws IOException { + if (perReaderTermState != null) { + final int ord; + if ((ord = perReaderTermState.lookup(reader)) != -1) { + final TermState state = perReaderTermState.get(ord); + if (state == null) { // term is not present in that reader + assert termNotInReader(reader, field, bytes); + return null; + } + return reader.termDocsEnum(reader.getDeletedDocs(), field, bytes, state); + } } - - return new TermScorer(this, docs, similarity, reader.norms(term.field())); + return reader.termDocsEnum(reader.getDeletedDocs(), field, bytes); } - + + private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException { + // only called from assert + final Terms terms = reader.terms(field); + return terms != null ? terms.docFreq(bytes) == 0 : true; + } + + @Override public Explanation explain(IndexReader reader, int doc) throws IOException { @@ -174,6 +196,17 @@ public TermQuery(Term t, int docFreq) { term = t; this.docFreq = docFreq; + perReaderTermState = null; + } + + /** Expert: constructs a TermQuery that will use the + * provided docFreq instead of looking up the docFreq + * against the searcher. */ + public TermQuery(Term t, PerReaderTermState states) { + assert states != null; + term = t; + docFreq = states.docFreq(); + perReaderTermState = states; } /** Returns the term of this query. */ Index: lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (working copy) @@ -25,9 +25,11 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PerReaderTermState; /** * Base rewrite method for collecting only the top terms @@ -78,12 +80,12 @@ this.termComp = termsEnum.getComparator(); // lazy init the initial ScoreTerm because comparator is not known on ctor: if (st == null) - st = new ScoreTerm(this.termComp); + st = new ScoreTerm(this.termComp, new PerReaderTermState(this.termStateReaderView)); boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class); } @Override - public boolean collect(BytesRef bytes) { + public boolean collect(BytesRef bytes) throws IOException { final float boost = boostAtt.getBoost(); // ignore uncompetetive hits if (stQueue.size() == maxSize) { @@ -94,23 +96,27 @@ return true; } ScoreTerm t = visitedTerms.get(bytes); + final TermState state = termsEnum.termState(); + assert state != null; if (t != null) { // if the term is already in the PQ, only update docFreq of term in PQ - t.docFreq += termsEnum.docFreq(); assert t.boost == boost : "boost should be equal in all segment TermsEnums"; + t.termState.register(state, reader); } else { // add new entry in PQ, we must clone the term, else it may get overwritten! st.bytes.copy(bytes); st.boost = boost; - st.docFreq = termsEnum.docFreq(); visitedTerms.put(st.bytes, st); + assert st.termState.docFreq() == 0; + st.termState.register(state, reader); stQueue.offer(st); // possibly drop entries from queue if (stQueue.size() > maxSize) { st = stQueue.poll(); visitedTerms.remove(st.bytes); + st.termState.clear(); // reset the termstate! } else { - st = new ScoreTerm(termComp); + st = new ScoreTerm(termComp, new PerReaderTermState(this.termStateReaderView)); } assert stQueue.size() <= maxSize : "the PQ size must be limited to maxSize"; // set maxBoostAtt with values to help FuzzyTermsEnum to optimize @@ -120,6 +126,7 @@ maxBoostAtt.setCompetitiveTerm(t.bytes); } } + return true; } }); @@ -130,8 +137,8 @@ ArrayUtil.quickSort(scoreTerms, scoreTermSortByTermComp); for (final ScoreTerm st : scoreTerms) { final Term term = placeholderTerm.createTerm(st.bytes); - assert reader.docFreq(term) == st.docFreq; - addClause(q, term, st.docFreq, query.getBoost() * st.boost); // add to query + assert reader.docFreq(term) == st.termState.docFreq() : "reader DF is " + reader.docFreq(term) + " vs " + st.termState.docFreq(); + addClause(q, term, st.termState.docFreq(), query.getBoost() * st.boost, st.termState); // add to query } query.incTotalNumberOfTerms(scoreTerms.length); return q; @@ -147,7 +154,7 @@ if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; - final TopTermsRewrite other = (TopTermsRewrite) obj; + final TopTermsRewrite other = (TopTermsRewrite) obj; if (size != other.size) return false; return true; } @@ -163,13 +170,12 @@ static final class ScoreTerm implements Comparable { public final Comparator termComp; - public final BytesRef bytes = new BytesRef(); public float boost; - public int docFreq; - - public ScoreTerm(Comparator termComp) { + public final PerReaderTermState termState; + public ScoreTerm(Comparator termComp, PerReaderTermState termState) { this.termComp = termComp; + this.termState = termState; } public int compareTo(ScoreTerm other) { Index: lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java =================================================================== --- lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (working copy) @@ -24,8 +24,10 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache.DocTermsIndex; import org.apache.lucene.util.ArrayUtil; @@ -284,21 +286,11 @@ } @Override - public void cacheCurrentTerm() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override public BytesRef term() throws IOException { return term; } @Override - public long ord() throws IOException { - return currentOrd; - } - - @Override public int docFreq() { throw new UnsupportedOperationException(); } @@ -317,6 +309,24 @@ public Comparator getComparator() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public SeekStatus seek(BytesRef term /*ignored*/, TermState state) throws IOException { + assert state != null && state instanceof PrefixCodedTermState; + return this.seek(((PrefixCodedTermState)state).ord); + } + + @Override + public TermState termState() throws IOException { + PrefixCodedTermState state = new PrefixCodedTermState() { + @Override + public int docFreq() { + throw new UnsupportedOperationException(); + } + }; + state.ord = currentOrd; + return state; + } } } } Index: lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (revision 1051521) +++ lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.search.TopTermsRewrite; import org.apache.lucene.search.ScoringRewrite; import org.apache.lucene.search.BooleanClause.Occur; // javadocs only +import org.apache.lucene.util.PerReaderTermState; /** * Wraps any {@link MultiTermQuery} as a {@link SpanQuery}, @@ -153,7 +154,7 @@ } @Override - protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost) { + protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) { final SpanTermQuery q = new SpanTermQuery(term); q.setBoost(boost); topLevel.addClause(q); @@ -202,7 +203,7 @@ } @Override - protected void addClause(SpanOrQuery topLevel, Term term, int docFreq, float boost) { + protected void addClause(SpanOrQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) { final SpanTermQuery q = new SpanTermQuery(term); q.setBoost(boost); topLevel.addClause(q); Index: lucene/src/java/org/apache/lucene/util/PerReaderTermState.java =================================================================== --- lucene/src/java/org/apache/lucene/util/PerReaderTermState.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/PerReaderTermState.java (revision 0) @@ -0,0 +1,202 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.WeakHashMap; + +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.TermsEnum.SeekStatus; + +/** + * Maintains a {@link IndexReader} {@link TermState} view over + * {@link IndexReader} instances containing a single term. The + * {@link PerReaderTermState} doesn't track if the given {@link TermState} + * objects are valid, neither if the {@link TermState} instances refer to the + * same terms in the associated readers. + * + * @lucene.experimental + */ +public final class PerReaderTermState { + + private final TermState[] states; + private int docFreq; + private final ReaderView view; + + /** + * Creates an empty {@link PerReaderTermState} from a {@link ReaderView} + */ + public PerReaderTermState(ReaderView view) { + docFreq = 0; + this.view = view; + states = new TermState[view.size()]; + } + + /** + * Creates a {@link PerReaderTermState} with an initial {@link TermState}, + * {@link IndexReader} pair. + */ + public PerReaderTermState(ReaderView view, TermState state, IndexReader reader) { + this(view); + register(state, reader); + } + + /** + * Creates a {@link PerReaderTermState} from an {@link IndexReader} and the + * given {@link Term} This method will gather all of the {@link IndexReader}s + * sub-readers and register each of the sub-readers in the + * {@link PerReaderTermState} is the term is present in the sub-reader. + */ + public static PerReaderTermState build(IndexReader topLevel, Term term) + throws IOException { + final List allSubReaders = new ArrayList(); + final String field = term.field(); + final BytesRef bytes = term.bytes(); + ReaderUtil.gatherSubReaders(allSubReaders, topLevel); + final PerReaderTermState perReaderTermState = new PerReaderTermState( + new ReaderView(allSubReaders)); + for (IndexReader indexReader : allSubReaders) { + final Fields fields = indexReader.fields(); + if (fields != null) { + final Terms terms = fields.terms(field); + if (terms != null) { + TermsEnum termsEnum = terms.iterator(); + if (SeekStatus.FOUND == termsEnum.seek(bytes, false)) { // no cache + // here! + final TermState termState = termsEnum.termState(); + perReaderTermState.register(termState, indexReader); + } + } + } + } + return perReaderTermState; + } + + /** + * Clears the {@link PerReaderTermState} internal state and removes all + * registered {@link TermState}s + */ + public void clear() { + docFreq = 0; + Arrays.fill(states, null); + } + + /** + * Registers and associates a {@link TermState} with an {@link IndexReader}. + */ + public void register(TermState state, IndexReader reader) { + assert state != null : "state must not be null"; + final int readerId = view.ord(reader); + assert readerId >= 0 && readerId < states.length; + assert states[readerId] == null : "state for reader: " + readerId + + " already registered"; + docFreq += state.docFreq(); + states[readerId] = state; + + } + + /** + * Returns a readers ordinal to use with {@link #get(int)} to obtain the + * {@link TermState} for that reader or -1 if this reader is not part + * of the view. + */ + public int lookup(IndexReader reader) { + return view.ord(reader); + } + + /** + * Returns the {@link TermState} for the given readers ord or null if no + * {@link TermState} for the reader was registered + * + * @param ord + * the readers ordinal to get the {@link TermState} for. + * @see #lookup(IndexReader) + * @return the {@link TermState} for the given readers ord or null if no + * {@link TermState} for the reader was registered + */ + public TermState get(int ord) { + assert ord >= 0 && ord < states.length; + return states[ord]; + } + + /** + * @return the accumulated document frequency of all {@link TermState} + * instances passed to {@link #register(TermState, IndexReader)}. + */ + public int docFreq() { + return docFreq; + } + + /** + * A {@link IndexReader} view that maps a given reader collection to an + * ordinal. This class is typically used to construct one or more + * {@link PerReaderTermState} instances to share the reader to ordinal mapping + * across multiple instances. + *

+ * The view maintains a reader to ordinal mapping with weak references such + * that {@link IndexReader} references are available for garbage collection + * once they are not in ordinary use anymore. + * + */ + public static final class ReaderView { + private final WeakHashMap readerMapping = new WeakHashMap(); + + /** + * Creates a new {@link ReaderView} + */ + public ReaderView(Collection readers) { + int i = 0; + for (IndexReader indexReader : readers) { + readerMapping.put(indexReader, Integer.valueOf(i++)); + } + } + + /** + * Returns the readers ordinal or -1 if the reader isn't in the + * view. + * + * @param reader + * the reader to look up. + * @return Returns the readers ordinal or -1 if the reader isn't in + * the view. + */ + public int ord(IndexReader reader) { + Integer id = readerMapping.get(reader); + return id == null ? -1 : id.intValue(); + } + + /** + * Returns the number of readers in the view + * + * @return the number of readers in the view + */ + public int size() { + return readerMapping.size(); + } + } + +} Index: lucene/src/test/org/apache/lucene/TestExternalCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/TestExternalCodecs.java (revision 1051521) +++ lucene/src/test/org/apache/lucene/TestExternalCodecs.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.util.*; +import org.apache.lucene.util.Bits; import org.apache.lucene.index.*; import org.apache.lucene.document.*; import org.apache.lucene.search.*; @@ -314,11 +315,6 @@ } @Override - public long ord() { - throw new UnsupportedOperationException(); - } - - @Override public BytesRef term() { // TODO: reuse BytesRef return new BytesRef(current); @@ -330,10 +326,6 @@ } @Override - public void cacheCurrentTerm() { - } - - @Override public DocsEnum docs(Bits skipDocs, DocsEnum reuse) { return new RAMDocsEnum(ramField.termToDocs.get(current), skipDocs); } @@ -342,6 +334,26 @@ public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) { return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), skipDocs); } + + @Override + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + assert term != null; + return seek(term); + } + + @Override + public TermState termState() throws IOException { + return new TermState() { + private int docFreq = RAMTermsEnum.this.docFreq(); + @Override + public int docFreq() { + return docFreq; + } + @Override + public void copy(TermState other) { + } + }; + } } private static class RAMDocsEnum extends DocsEnum { Index: lucene/src/test/org/apache/lucene/index/TestFlex.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFlex.java (revision 1051521) +++ lucene/src/test/org/apache/lucene/index/TestFlex.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.store.*; import org.apache.lucene.analysis.*; import org.apache.lucene.document.*; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.util.*; public class TestFlex extends LuceneTestCase { @@ -71,7 +72,9 @@ IndexReader r = w.getReader(); TermsEnum terms = r.getSequentialSubReaders()[0].fields().terms("f").iterator(); assertTrue(terms.next() != null); - assertEquals(0, terms.ord()); + TermState termState = terms.termState(); + assertTrue(termState instanceof PrefixCodedTermState); + assertEquals(0, ((PrefixCodedTermState)termState).ord()); r.close(); w.close(); d.close(); Index: lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (revision 1051521) +++ lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.store.Directory; @@ -82,11 +83,17 @@ assertNotNull(terms.next()); assertEquals("aaa", terms.term().utf8ToString()); assertNotNull(terms.next()); - long ordB = terms.ord(); + final BytesRef currentTerm = terms.term(); + final TermState state = terms.termState(); assertEquals("bbb", terms.term().utf8ToString()); assertNull(terms.next()); - - assertEquals(TermsEnum.SeekStatus.FOUND, terms.seek(ordB)); + assertTrue(state instanceof PrefixCodedTermState); + PrefixCodedTermState stdTermState = ((PrefixCodedTermState)state); + if (random.nextBoolean()) { + assertEquals(TermsEnum.SeekStatus.FOUND, terms.seek(stdTermState.ord())); + } else { + assertEquals(TermsEnum.SeekStatus.FOUND, terms.seek(currentTerm, stdTermState)); + } assertEquals("bbb", terms.term().utf8ToString()); reader.close(); } Index: lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (revision 1051521) +++ lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.store.Directory; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -309,11 +310,10 @@ hits=indexSearcher1.search(query, null, 1000).scoreDocs; assertEquals(message, 2, hits.length); - // Store the scores for use later float[] scores={ hits[0].score, hits[1].score }; - assertTrue(message, scores[0] > scores[1]); + assertTrue(message + " " + Arrays.toString(scores), scores[0] > scores[1]); indexSearcher1.close(); ramDirectory1.close(); Index: lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java =================================================================== --- lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java (revision 1051521) +++ lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java (working copy) @@ -43,6 +43,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PrefixCodedTermState; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexInput; @@ -958,11 +959,7 @@ int ord = 0; while((term = termsEnum.next()) != null) { if (ord == 0) { - try { - termsEnum.ord(); - } catch (UnsupportedOperationException uoe) { - storeOrd = false; - } + storeOrd = termsEnum.termState() instanceof PrefixCodedTermState; } final int output; if (storeOrd) { @@ -1035,7 +1032,7 @@ assertEquals(termsEnum.term(), fstEnum.current().input); if (storeOrd) { // fst stored the ord - assertEquals(termsEnum.ord(), ((Long) fstEnum.current().output).longValue()); + assertEquals(((PrefixCodedTermState)termsEnum.termState()).ord(), ((Long) fstEnum.current().output).longValue()); } else { // fst stored the docFreq assertEquals(termsEnum.docFreq(), (int) (((Long) fstEnum.current().output).longValue())); Index: solr/src/java/org/apache/solr/request/UnInvertedField.java =================================================================== --- solr/src/java/org/apache/solr/request/UnInvertedField.java (revision 1051521) +++ solr/src/java/org/apache/solr/request/UnInvertedField.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.MultiFields; @@ -1001,10 +1002,6 @@ return tenum.docFreq(); } - @Override - public void cacheCurrentTerm() { - throw new UnsupportedOperationException(); - } public BytesRef skipTo(BytesRef target) throws IOException { @@ -1091,11 +1088,6 @@ } @Override - public long ord() { - throw new UnsupportedOperationException(); - } - - @Override public SeekStatus seek(long ord) { throw new UnsupportedOperationException(); } @@ -1114,6 +1106,18 @@ public SeekStatus seek(BytesRef target, boolean useCache) { throw new UnsupportedOperationException(); } + + + @Override + public SeekStatus seek(BytesRef term, TermState state) throws IOException { + throw new UnsupportedOperationException(); + } + + + @Override + public TermState termState() throws IOException { + throw new UnsupportedOperationException(); + } }