Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java =================================================================== --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java (revision 1036841) +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java (working copy) @@ -19,9 +19,12 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; + +import java.io.IOException; import java.util.Arrays; import java.util.Comparator; @@ -91,10 +94,6 @@ } @Override - public void cacheCurrentTerm() { - } - - @Override public BytesRef term() { return br; } @@ -129,5 +128,22 @@ public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + return docs(skipDocs, reuse); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + return docsAndPositions(skipDocs, reuse); + } + + @Override + public TermState getTermState() throws IOException { + return null; + } } Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1036841) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -38,6 +38,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Fields; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.FieldsEnum; @@ -877,10 +878,6 @@ } @Override - public void cacheCurrentTerm() { - } - - @Override public long ord() { return termUpto; } @@ -910,6 +907,23 @@ public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + return docs(skipDocs, reuse); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + return docsAndPositions(skipDocs, reuse); + } + + @Override + public TermState getTermState() throws IOException { + return null; + } } private class MemoryDocsEnum extends DocsEnum { Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (revision 1036841) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (working copy) @@ -112,7 +112,6 @@ docsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; return ret; Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java (revision 1036841) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java (working copy) @@ -35,9 +35,9 @@ public AppendingTermsDictReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize, - Comparator termComp, int termsCacheSize, String codecId) throws IOException { + Comparator termComp, String codecId) throws IOException { super(indexReader, dir, fieldInfos, segment, postingsReader, readBufferSize, - termComp, termsCacheSize, codecId); + termComp, codecId); } @Override Index: lucene/src/java/org/apache/lucene/index/FilterIndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -130,11 +130,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - in.cacheCurrentTerm(); - } - - @Override public SeekStatus seek(long ord) throws IOException { return in.seek(ord); } @@ -173,6 +168,23 @@ public Comparator getComparator() throws IOException { return in.getComparator(); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + return in.docs(skipDocs, reuse, termState); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + return in.docsAndPositions(skipDocs, reuse, termState); + } + + @Override + public TermState getTermState() throws IOException { + return in.getTermState(); + } } /** Base class for filtering {@link DocsEnum} implementations. */ Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1066,7 +1066,38 @@ return null; } } + + //nocommit - javadoc + public DocsEnum termDocsEnum(Bits skipDocs, TermState state) throws IOException { + assert state != null; + final Fields fields = fields(); + if (fields == null) { + return null; + } + final Terms terms = fields.terms(state.field()); + if (terms != null) { + return terms.docs(skipDocs, state, null); + } else { + return null; + } + } + + //nocommit - javadoc + public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, TermState state) throws IOException { + assert state != null; + final Fields fields = fields(); + if (fields == null) { + return null; + } + final Terms terms = fields.terms(state.field()); + if (terms != null) { + return terms.docsAndPositions(skipDocs, state, null); + } else { + return null; + } + } + /** Deletes the document numbered docNum. Once a document is * deleted it will not appear in TermDocs or TermPositions enumerations. * Attempts to read its field with the {@link #document} Index: lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (working copy) @@ -91,13 +91,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - for(int i=0;i getComparator() { return termComp; } @@ -434,4 +427,21 @@ } } } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TermState getTermState() throws IOException { + return null; + } } Index: lucene/src/java/org/apache/lucene/index/TermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermState.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/TermState.java (working copy) @@ -1,56 +1,42 @@ -package org.apache.lucene.index.codecs; +package org.apache.lucene.index; + /** * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ -import org.apache.lucene.index.DocsEnum; // for javadocs - -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs - /** - * Holds all state required for {@link StandardPostingsReader} - * to produce a {@link DocsEnum} without re-seeking the - * terms dict. * @lucene.experimental */ +public abstract class TermState implements Cloneable { -public class TermState implements Cloneable { - public long ord; // ord for this term - public long filePointer; // fp into the terms dict primary file (_X.tis) - public int docFreq; // how many docs have this term + public abstract void copy(TermState other); - public void copy(TermState other) { - ord = other.ord; - filePointer = other.filePointer; - docFreq = other.docFreq; - } + public abstract int docFreq(); - @Override + public abstract long ord(); + + public abstract String field(); + public Object clone() { try { return super.clone(); - } catch (CloneNotSupportedException cnse) { + } catch (CloneNotSupportedException e) { // should not happen - throw new RuntimeException(cnse); + throw new RuntimeException(e); } } - - @Override - public String toString() { - return "tis.fp=" + filePointer + " docFreq=" + docFreq + " ord=" + ord; - } } Index: lucene/src/java/org/apache/lucene/index/Terms.java =================================================================== --- lucene/src/java/org/apache/lucene/index/Terms.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/Terms.java (working copy) @@ -79,7 +79,19 @@ return null; } } + + //nocommit - javadoc + public DocsEnum docs(Bits skipDocs, TermState termState, DocsEnum reuse) throws IOException { + final TermsEnum termsEnum = getThreadTermsEnum(); + return termsEnum.docs(skipDocs, reuse, termState); + } + //nocommit - javadoc + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, TermState termState, DocsAndPositionsEnum reuse) throws IOException { + final TermsEnum termsEnum = getThreadTermsEnum(); + return termsEnum.docsAndPositions(skipDocs, reuse, termState); + } + public long getUniqueTermCount() throws IOException { throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()"); } Index: lucene/src/java/org/apache/lucene/index/TermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermsEnum.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/TermsEnum.java (working copy) @@ -98,7 +98,14 @@ * first time, after next() returns null or seek returns * {@link SeekStatus#END}.*/ public abstract int docFreq(); + + //nocommit - javadoc + public abstract DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) throws IOException; + //nocommit - javadoc + public abstract DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, TermState termState) throws IOException; + + /** Get {@link DocsEnum} for the current term. Do not * call this before calling {@link #next} or {@link * #seek} for the first time. This method will not @@ -115,7 +122,10 @@ * only return null if positions were not indexed into * the postings by this codec. */ public abstract DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; - + + //nocommit - javadoc + public abstract TermState getTermState() throws IOException; + /** Return the {@link BytesRef} Comparator used to sort * terms provided by the iterator. This may return * null if there are no terms. Callers may invoke this @@ -123,10 +133,6 @@ * instance & reuse it. */ public abstract Comparator getComparator() throws IOException; - /** Optional optimization hint: informs the codec that the - * current term is likely to be re-seek'd-to soon. */ - public abstract void cacheCurrentTerm() throws IOException; - /** An empty TermsEnum for quickly returning an empty instance e.g. * in {@link org.apache.lucene.search.MultiTermQuery} *

Please note: This enum should be unmodifiable, @@ -142,9 +148,6 @@ public SeekStatus seek(long ord) { return SeekStatus.END; } @Override - public void cacheCurrentTerm() {} - - @Override public BytesRef term() { throw new IllegalStateException("this method should never be called"); } @@ -183,5 +186,22 @@ public synchronized AttributeSource attributes() { return super.attributes(); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + throw new IllegalStateException("this method should never be called"); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + throw new IllegalStateException("this method should never be called"); + } + + @Override + public TermState getTermState() throws IOException { + throw new IllegalStateException("this method should never be called"); + } }; } Index: lucene/src/java/org/apache/lucene/index/codecs/PerReaderTermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PerReaderTermState.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/PerReaderTermState.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.IdentityHashMap; +import java.util.Map; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.TermState; + +/** + * + * @lucene.experimental + */ +public class PerReaderTermState { + + private final Map mapping = new IdentityHashMap(); + + public void set(TermState state, IndexReader reader) { + mapping.put(reader, state); + } + + public TermState get(IndexReader reader) { + return mapping.get(reader); + } +} Property changes on: lucene/src/java/org/apache/lucene/index/codecs/PerReaderTermState.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (working copy) @@ -42,17 +42,17 @@ public abstract void init(IndexInput termsIn) throws IOException; /** Return a newly created empty TermState */ - public abstract TermState newTermState() throws IOException; + public abstract StandardTermState newTermState() throws IOException; - public abstract void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState state, boolean isIndexTerm) throws IOException; + public abstract void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState state, boolean isIndexTerm) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsEnum docs(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsEnum reuse) throws IOException; + public abstract DocsEnum docs(FieldInfo fieldInfo, StandardTermState state, Bits skipDocs, DocsEnum reuse) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; + public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, StandardTermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; public abstract void close() throws IOException; } Index: lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.Directory; @@ -67,56 +68,18 @@ // Comparator that orders our terms private final Comparator termComp; - // Caches the most recently looked-up field + terms: - private final DoubleBarrelLRUCache termsCache; - // Reads the terms index private TermsIndexReaderBase indexReader; // keeps the dirStart offset protected long dirOffset; - // Used as key for the terms cache - private static class FieldAndTerm extends DoubleBarrelLRUCache.CloneableKey { - String field; - BytesRef term; - - public FieldAndTerm() { - } - - public FieldAndTerm(String field, BytesRef term) { - this.field = field; - this.term = new BytesRef(term); - } - - public FieldAndTerm(FieldAndTerm other) { - field = other.field; - term = new BytesRef(other.term); - } - - @Override - public boolean equals(Object _other) { - FieldAndTerm other = (FieldAndTerm) _other; - return other.field == field && term.bytesEquals(other.term); - } - - @Override - public Object clone() { - return new FieldAndTerm(this); - } - - @Override - public int hashCode() { - return field.hashCode() * 31 + term.hashCode(); - } - } public PrefixCodedTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize, - Comparator termComp, int termsCacheSize, String codecId) + Comparator termComp, String codecId) throws IOException { this.postingsReader = postingsReader; - termsCache = new DoubleBarrelLRUCache(termsCacheSize); this.termComp = termComp; @@ -285,16 +248,15 @@ private class SegmentTermsEnum extends TermsEnum { private final IndexInput in; private final DeltaBytesReader bytesReader; - private final TermState state; + private final StandardTermState state; private boolean seekPending; private final TermsIndexReaderBase.TermsIndexResult indexResult = new TermsIndexReaderBase.TermsIndexResult(); - private final FieldAndTerm fieldTerm = new FieldAndTerm(); - + private final String field; SegmentTermsEnum() throws IOException { in = (IndexInput) PrefixCodedTermsReader.this.in.clone(); in.seek(termsStartPointer); bytesReader = new DeltaBytesReader(in); - fieldTerm.field = fieldInfo.name; + field = fieldInfo.name; state = postingsReader.newTermState(); state.ord = -1; } @@ -304,37 +266,13 @@ return termComp; } - @Override - public void cacheCurrentTerm() { - TermState stateCopy = (TermState) state.clone(); - stateCopy.filePointer = in.getFilePointer(); - termsCache.put(new FieldAndTerm(fieldInfo.name, bytesReader.term), - stateCopy); - } - /** Seeks until the first term that's >= the provided * text; returns SeekStatus.FOUND if the exact term * is found, SeekStatus.NOT_FOUND if a different term * was found, SeekStatus.END if we hit EOF */ @Override public SeekStatus seek(BytesRef term, boolean useCache) throws IOException { - // Check cache - fieldTerm.term = term; - TermState cachedState; - if (useCache) { - cachedState = termsCache.get(fieldTerm); - if (cachedState != null) { - state.copy(cachedState); - seekPending = true; - bytesReader.term.copy(term); - return SeekStatus.FOUND; - } - } else { - cachedState = null; - } - boolean doSeek = true; - if (state.ord != -1) { // we are positioned @@ -387,16 +325,6 @@ while(next() != null) { final int cmp = termComp.compare(bytesReader.term, term); if (cmp == 0) { - - if (doSeek && useCache) { - // Store in cache - FieldAndTerm entryKey = new FieldAndTerm(fieldTerm); - cachedState = (TermState) state.clone(); - // this is fp after current term - cachedState.filePointer = in.getFilePointer(); - termsCache.put(entryKey, cachedState); - } - return SeekStatus.FOUND; } else if (cmp > 0) { return SeekStatus.NOT_FOUND; @@ -507,6 +435,34 @@ return postingsReader.docsAndPositions(fieldInfo, state, skipDocs, reuse); } } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + assert termState.field() == fieldInfo.name; + DocsEnum docsEnum = postingsReader.docs(fieldInfo, (StandardTermState) termState, skipDocs, reuse); + assert docsEnum != null; + return docsEnum; + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + if (fieldInfo.omitTermFreqAndPositions) { + return null; + } else { + assert termState.field() == fieldInfo.name; + return postingsReader.docsAndPositions(fieldInfo, (StandardTermState) termState, skipDocs, reuse); + } + } + + @Override + public TermState getTermState() throws IOException { + StandardTermState termState = (StandardTermState) state.clone(); + termState.field = field; + termState.filePointer = in.getFilePointer(); + return termState; + } } } } Index: lucene/src/java/org/apache/lucene/index/codecs/StandardTermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/StandardTermState.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/StandardTermState.java (revision 0) @@ -0,0 +1,69 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.DocsEnum; // for javadocs +import org.apache.lucene.index.TermState; + +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs + +/** + * Holds all state required for {@link StandardPostingsReader} + * to produce a {@link DocsEnum} without re-seeking the + * terms dict. + * @lucene.experimental + */ +public class StandardTermState extends TermState { + public long ord; // ord for this term + public long filePointer; // fp into the terms dict primary file (_X.tis) + public int docFreq; // how many docs have this term + public String field; + + public void copy(TermState other) { + final StandardTermState state = (StandardTermState) other; + ord = state.ord; + filePointer = state.filePointer; + docFreq = state.docFreq; + field = state.field; + } + + @Override + public Object clone() { + return super.clone(); + } + + @Override + public String toString() { + return "tis.fp=" + filePointer + " docFreq=" + docFreq + " ord=" + ord; + } + + @Override + public int docFreq() { + return docFreq; + } + + @Override + public long ord() { + return ord; + } + + @Override + public String field() { + return field; + } +} Property changes on: lucene/src/java/org/apache/lucene/index/codecs/StandardTermState.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/index/codecs/TermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/TermState.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/TermState.java (working copy) @@ -1,56 +0,0 @@ -package org.apache.lucene.index.codecs; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.DocsEnum; // for javadocs - -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs - -/** - * Holds all state required for {@link StandardPostingsReader} - * to produce a {@link DocsEnum} without re-seeking the - * terms dict. - * @lucene.experimental - */ - -public class TermState implements Cloneable { - public long ord; // ord for this term - public long filePointer; // fp into the terms dict primary file (_X.tis) - public int docFreq; // how many docs have this term - - public void copy(TermState other) { - ord = other.ord; - filePointer = other.filePointer; - docFreq = other.docFreq; - } - - @Override - public Object clone() { - try { - return super.clone(); - } catch (CloneNotSupportedException cnse) { - // should not happen - throw new RuntimeException(cnse); - } - } - - @Override - public String toString() { - return "tis.fp=" + filePointer + " docFreq=" + docFreq + " ord=" + ord; - } -} Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (working copy) @@ -33,6 +33,7 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.CompoundFileReader; @@ -739,11 +740,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - getTermsDict().cacheCurrentTerm(termEnum); - } - - @Override public SeekStatus seek(long ord) throws IOException { throw new UnsupportedOperationException(); } @@ -968,6 +964,23 @@ } return docsPosEnum.reset(termEnum, skipDocs); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TermState getTermState() throws IOException { + return null; + } } private final class PreDocsEnum extends DocsEnum { Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (working copy) @@ -137,7 +137,6 @@ pulsingReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; return ret; Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (working copy) @@ -22,7 +22,8 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.codecs.TermState; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.codecs.StandardTermState; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl.Document; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl.Position; @@ -57,11 +58,12 @@ wrappedPostingsReader.init(termsIn); } - private static class PulsingTermState extends TermState { + private static class PulsingTermState extends StandardTermState { private Document docs[]; - private TermState wrappedTermState; + private StandardTermState wrappedTermState; private boolean pendingIndexTerm; + @Override public Object clone() { PulsingTermState clone; clone = (PulsingTermState) super.clone(); @@ -72,10 +74,11 @@ clone.docs[i] = (Document) doc.clone(); } } - clone.wrappedTermState = (TermState) wrappedTermState.clone(); + clone.wrappedTermState = (StandardTermState) wrappedTermState.clone(); return clone; } + @Override public void copy(TermState _other) { super.copy(_other); PulsingTermState other = (PulsingTermState) _other; @@ -90,7 +93,7 @@ } @Override - public TermState newTermState() throws IOException { + public StandardTermState newTermState() throws IOException { PulsingTermState state = new PulsingTermState(); state.wrappedTermState = wrappedPostingsReader.newTermState(); state.docs = new Document[maxPulsingDocFreq]; @@ -98,8 +101,7 @@ } @Override - public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState _termState, boolean isIndexTerm) throws IOException { - + public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState _termState, boolean isIndexTerm) throws IOException { PulsingTermState termState = (PulsingTermState) _termState; termState.pendingIndexTerm |= isIndexTerm; @@ -182,7 +184,7 @@ // TODO: we could actually reuse, by having TL that // holds the last wrapped reuse, and vice-versa @Override - public DocsEnum docs(FieldInfo field, TermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo field, StandardTermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { PulsingTermState termState = (PulsingTermState) _termState; if (termState.docFreq <= maxPulsingDocFreq) { if (reuse instanceof PulsingDocsEnum) { @@ -202,7 +204,7 @@ // TODO: -- not great that we can't always reuse @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo field, TermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo field, StandardTermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { PulsingTermState termState = (PulsingTermState) _termState; if (termState.docFreq <= maxPulsingDocFreq) { if (reuse instanceof PulsingDocsAndPositionsEnum) { Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (working copy) @@ -25,8 +25,9 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.TermState; +import org.apache.lucene.index.codecs.StandardTermState; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; @@ -129,18 +130,20 @@ } } - private static class SepTermState extends TermState { + private static class SepTermState extends StandardTermState { // We store only the seek point to the docs file because // the rest of the info (freqIndex, posIndex, etc.) is // stored in the docs file: IntIndexInput.Index docIndex; - + + @Override public Object clone() { SepTermState other = (SepTermState) super.clone(); other.docIndex = (IntIndexInput.Index) docIndex.clone(); return other; } + @Override public void copy(TermState _other) { super.copy(_other); SepTermState other = (SepTermState) _other; @@ -154,19 +157,19 @@ } @Override - public TermState newTermState() throws IOException { + public StandardTermState newTermState() throws IOException { final SepTermState state = new SepTermState(); state.docIndex = docIn.index(); return state; } @Override - public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState termState, boolean isIndexTerm) throws IOException { + public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState termState, boolean isIndexTerm) throws IOException { ((SepTermState) termState).docIndex.read(termsIn, isIndexTerm); } @Override - public DocsEnum docs(FieldInfo fieldInfo, TermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, StandardTermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { final SepTermState termState = (SepTermState) _termState; SepDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SepDocsEnum)) { @@ -185,7 +188,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, StandardTermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { assert !fieldInfo.omitTermFreqAndPositions; final SepTermState termState = (SepTermState) _termState; SepDocsAndPositionsEnum postingsEnum; Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; @@ -186,10 +187,6 @@ } @Override - public void cacheCurrentTerm() { - } - - @Override public BytesRef next() throws IOException { assert !ended; @@ -284,8 +281,25 @@ } return docsAndPositionsEnum.reset(docsStart, skipDocs); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + return docs(skipDocs, reuse); + } @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + return docsAndPositions(skipDocs, reuse); + } + + @Override + public TermState getTermState() throws IOException { + return null; + } + + @Override public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (working copy) @@ -80,8 +80,6 @@ } } - public final static int TERMS_CACHE_SIZE = 1024; - @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); @@ -111,7 +109,6 @@ postings, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - TERMS_CACHE_SIZE, state.codecId); success = true; return ret; Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (working copy) @@ -26,8 +26,9 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.TermState; +import org.apache.lucene.index.codecs.StandardTermState; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -82,11 +83,12 @@ maxSkipLevels = termsIn.readInt(); } - private static class DocTermState extends TermState { + private static class DocTermState extends StandardTermState { long freqOffset; long proxOffset; int skipOffset; - + + @Override public Object clone() { DocTermState other = (DocTermState) super.clone(); other.freqOffset = freqOffset; @@ -95,6 +97,7 @@ return other; } + @Override public void copy(TermState _other) { super.copy(_other); DocTermState other = (DocTermState) _other; @@ -109,7 +112,7 @@ } @Override - public TermState newTermState() { + public StandardTermState newTermState() { return new DocTermState(); } @@ -127,9 +130,8 @@ } @Override - public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState termState, boolean isIndexTerm) + public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState termState, boolean isIndexTerm) throws IOException { - final DocTermState docTermState = (DocTermState) termState; if (isIndexTerm) { @@ -154,7 +156,7 @@ } @Override - public DocsEnum docs(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs, DocsEnum reuse) throws IOException { SegmentDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SegmentDocsEnum)) { docsEnum = new SegmentDocsEnum(freqIn); @@ -171,7 +173,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { if (fieldInfo.omitTermFreqAndPositions) { return null; } Index: lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (working copy) @@ -21,9 +21,15 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray; class ConstantScoreAutoRewrite extends TermCollectingRewrite { @@ -71,8 +77,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/) { - topLevel.add(new TermQuery(term, docFreq), BooleanClause.Occur.SHOULD); + protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/, PerReaderTermState states) { + topLevel.add(new TermQuery(term, docFreq, states), BooleanClause.Occur.SHOULD); } @Override @@ -98,9 +104,10 @@ final BytesRefHash pendingTerms = col.pendingTerms; final int sort[] = pendingTerms.sort(col.termsEnum.getComparator()); for(int i = 0; i < size; i++) { + final int pos = sort[i]; // docFreq is not used for constant score here, we pass 1 // to explicitely set a fake value, so it's not calculated - addClause(bq, placeholderTerm.createTerm(pendingTerms.get(sort[i], new BytesRef())), 1, 1.0f); + addClause(bq, placeholderTerm.createTerm(pendingTerms.get(pos, new BytesRef())), 1, 1.0f, col.array.termState[pos]); } // Strip scores final Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq)); @@ -123,12 +130,22 @@ @Override public boolean collect(BytesRef bytes) throws IOException { - pendingTerms.add(bytes); + int pos = pendingTerms.add(bytes); docVisitCount += termsEnum.docFreq(); if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) { hasCutOff = true; return false; } + + final TermState termState = termsEnum.getTermState(); + if (pos < 0) { + pos = (-pos)-1; + } else { + array.termState[pos] = new PerReaderTermState(); + } + if (termState != null) { + array.termState[pos].set(termState, reader); + } return true; } @@ -137,7 +154,8 @@ TermsEnum termsEnum; final int docCountCutoff, termCountLimit; - final BytesRefHash pendingTerms = new BytesRefHash(); + final TermStateByteStart array = new TermStateByteStart(16); + final BytesRefHash pendingTerms = new BytesRefHash(new ByteBlockPool(new ByteBlockPool.DirectAllocator()), 16, array); } @Override @@ -166,4 +184,36 @@ return true; } + + /** Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq */ + static final class TermStateByteStart extends DirectBytesStartArray { + PerReaderTermState[] termState; + + public TermStateByteStart(int initSize) { + super(initSize); + } + + @Override + public int[] init() { + final int[] ord = super.init(); + termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJ_REF)]; + assert termState.length >= ord.length; + return ord; + } + + @Override + public int[] grow() { + final int[] ord = super.grow(); + termState = ArrayUtil.grow(PerReaderTermState.class, termState, ord.length); + assert termState.length >= ord.length; + return ord; + } + + @Override + public int[] clear() { + termState = null; + return super.clear(); + } + + } } Index: lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.MultiFields; @@ -177,12 +178,27 @@ assert tenum != null; return tenum.docsAndPositions(bits, reuse); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + assert tenum != null; + return tenum.docs(skipDocs, reuse, termState); + } @Override - public void cacheCurrentTerm() throws IOException { - tenum.cacheCurrentTerm(); + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + assert tenum != null; + return tenum.docsAndPositions(skipDocs, reuse, termState); } - + + @Override + public TermState getTermState() throws IOException { + assert tenum != null; + return tenum.getTermState(); + } + @SuppressWarnings("fallthrough") @Override public BytesRef next() throws IOException { Index: lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeImpl; @@ -246,11 +247,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - actualEnum.cacheCurrentTerm(); - } - - @Override public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { return actualEnum.docs(skipDocs, reuse); } @@ -262,6 +258,23 @@ } @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + return actualEnum.docs(skipDocs, reuse, termState); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + return actualEnum.docsAndPositions(skipDocs, reuse, termState); + } + + @Override + public TermState getTermState() throws IOException { + return actualEnum.getTermState(); + } + + @Override public Comparator getComparator() throws IOException { return actualEnum.getComparator(); } Index: lucene/src/java/org/apache/lucene/search/MultiTermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.util.AttributeSource; @@ -158,8 +159,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost) { - final TermQuery tq = new TermQuery(term, docCount); + protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) { + final TermQuery tq = new TermQuery(term, docCount, states); tq.setBoost(boost); topLevel.add(tq, BooleanClause.Occur.SHOULD); } @@ -199,8 +200,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost) { - final Query q = new ConstantScoreQuery(new QueryWrapperFilter(new TermQuery(term, docFreq))); + protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) { + final Query q = new ConstantScoreQuery(new QueryWrapperFilter(new TermQuery(term, docFreq, states))); q.setBoost(boost); topLevel.add(q, BooleanClause.Occur.SHOULD); } Index: lucene/src/java/org/apache/lucene/search/ScoringRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (working copy) @@ -28,7 +28,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.search.MultiTermQuery.RewriteMethod; import org.apache.lucene.util.ArrayUtil; @@ -62,11 +64,12 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost) { - final TermQuery tq = new TermQuery(term, docCount); + protected void addClause(BooleanQuery topLevel, Term term, int docCount, + float boost, PerReaderTermState states) { + final TermQuery tq = new TermQuery(term, docCount, states); tq.setBoost(boost); - topLevel.add(tq, BooleanClause.Occur.SHOULD); - } + topLevel.add(tq, BooleanClause.Occur.SHOULD); + } @Override protected void checkMaxClauseCount(int count) { @@ -77,7 +80,7 @@ // Make sure we are still a singleton even after deserializing protected Object readResolve() { return SCORING_BOOLEAN_QUERY_REWRITE; - } + } }; /** Like {@link #SCORING_BOOLEAN_QUERY_REWRITE} except @@ -125,11 +128,12 @@ final int sort[] = col.terms.sort(col.termsEnum.getComparator()); final int[] docFreq = col.array.docFreq; final float[] boost = col.array.boost; + final PerReaderTermState[] termStates = col.array.termState; for (int i = 0; i < size; i++) { final int pos = sort[i]; final Term term = placeholderTerm.createTerm(col.terms.get(pos, new BytesRef())); assert reader.docFreq(term) == docFreq[pos]; - addClause(result, term, docFreq[pos], query.getBoost() * boost[pos]); + addClause(result, term, docFreq[pos], query.getBoost() * boost[pos], termStates[pos]); } } query.incTotalNumberOfTerms(size); @@ -152,15 +156,22 @@ @Override public boolean collect(BytesRef bytes) throws IOException { final int e = terms.add(bytes); + final TermState state = termsEnum.getTermState(); if (e < 0 ) { // duplicate term: update docFreq final int pos = (-e)-1; array.docFreq[pos] += termsEnum.docFreq(); + if(state != null) + array.termState[pos].set(state, reader); assert array.boost[pos] == boostAtt.getBoost() : "boost should be equal in all segment TermsEnums"; } else { // new entry: we populate the entry initially array.docFreq[e] = termsEnum.docFreq(); array.boost[e] = boostAtt.getBoost(); + array.termState[e] = new PerReaderTermState(); + if(state != null) { + array.termState[e].set(state, reader); + } ScoringRewrite.this.checkMaxClauseCount(terms.size()); } return true; @@ -171,6 +182,7 @@ static final class TermFreqBoostByteStart extends DirectBytesStartArray { int[] docFreq; float[] boost; + PerReaderTermState[] termState; public TermFreqBoostByteStart(int initSize) { super(initSize); @@ -181,7 +193,8 @@ final int[] ord = super.init(); boost = new float[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_FLOAT)]; docFreq = new int[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_INT)]; - assert boost.length >= ord.length && docFreq.length >= ord.length; + termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJ_REF)]; + assert termState.length >= ord.length && boost.length >= ord.length && docFreq.length >= ord.length; return ord; } @@ -190,7 +203,8 @@ final int[] ord = super.grow(); docFreq = ArrayUtil.grow(docFreq, ord.length); boost = ArrayUtil.grow(boost, ord.length); - assert boost.length >= ord.length && docFreq.length >= ord.length; + termState = ArrayUtil.grow(PerReaderTermState.class, termState, ord.length); + assert termState.length >= ord.length && boost.length >= ord.length && docFreq.length >= ord.length; return ord; } @@ -198,6 +212,7 @@ public int[] clear() { boost = null; docFreq = null; + termState = null; return super.clear(); } Index: lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java (working copy) @@ -28,6 +28,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ReaderUtil; @@ -38,8 +39,13 @@ protected abstract Q getTopLevelQuery() throws IOException; /** Add a MultiTermQuery term to the top-level query */ - protected abstract void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException; + protected final void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException { + addClause(topLevel, term, docCount, boost, null); + } + protected abstract void addClause(Q topLevel, Term term, int docCount, float boost, PerReaderTermState states) throws IOException; + + protected final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException { final List subReaders = new ArrayList(); ReaderUtil.gatherSubReaders(subReaders, reader); @@ -71,9 +77,9 @@ lastTermComp = newTermComp; collector.setNextEnum(termsEnum); + collector.setNextReader(r); BytesRef bytes; while ((bytes = termsEnum.next()) != null) { - termsEnum.cacheCurrentTerm(); if (!collector.collect(bytes)) return; // interrupt whole term collection, so also don't iterate other subReaders } @@ -81,6 +87,8 @@ } protected static abstract class TermCollector { + protected IndexReader reader; + /** attributes used for communication with the enum */ public final AttributeSource attributes = new AttributeSource(); @@ -89,5 +97,9 @@ /** the next segment's {@link TermsEnum} that is used to collect terms */ public abstract void setNextEnum(TermsEnum termsEnum) throws IOException; + + public void setNextReader(IndexReader reader) { + this.reader = reader; + } } } Index: lucene/src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermQuery.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -23,6 +23,8 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.search.Explanation.IDFExplanation; import org.apache.lucene.util.ToStringUtils; @@ -32,6 +34,7 @@ public class TermQuery extends Query { private final Term term; private final int docFreq; + private final PerReaderTermState perReaderTermState; private class TermWeight extends Weight { private final Similarity similarity; @@ -76,10 +79,15 @@ @Override public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), - term.field(), - term.bytes()); - + TermState state; + final DocsEnum docs; + if(perReaderTermState != null && (state = perReaderTermState.get(reader)) != null) { + docs = reader.termDocsEnum(reader.getDeletedDocs(), state); + } else { + docs = reader.termDocsEnum(reader.getDeletedDocs(), + term.field(), + term.bytes()); + } if (docs == null) { return null; } @@ -174,7 +182,17 @@ public TermQuery(Term t, int docFreq) { term = t; this.docFreq = docFreq; + this.perReaderTermState = null; } + + /** Expert: constructs a TermQuery that will use the + * provided docFreq instead of looking up the docFreq + * against the searcher. */ + public TermQuery(Term t, int docFreq, PerReaderTermState states) { + term = t; + this.docFreq = docFreq; + this.perReaderTermState = states; + } /** Returns the term of this query. */ public Term getTerm() { return term; } Index: lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (working copy) @@ -25,7 +25,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -83,7 +85,7 @@ } @Override - public boolean collect(BytesRef bytes) { + public boolean collect(BytesRef bytes) throws IOException { final float boost = boostAtt.getBoost(); // ignore uncompetetive hits if (stQueue.size() == maxSize) { @@ -94,17 +96,23 @@ return true; } ScoreTerm t = visitedTerms.get(bytes); + final TermState state = termsEnum.getTermState(); if (t != null) { // if the term is already in the PQ, only update docFreq of term in PQ t.docFreq += termsEnum.docFreq(); assert t.boost == boost : "boost should be equal in all segment TermsEnums"; + if (state != null) { + t.termState.set(state, reader); + } } else { // add new entry in PQ, we must clone the term, else it may get overwritten! st.bytes.copy(bytes); st.boost = boost; st.docFreq = termsEnum.docFreq(); visitedTerms.put(st.bytes, st); - stQueue.offer(st); + if(stQueue.offer(st) && state != null) { + st.termState.set(state, reader); + } // possibly drop entries from queue if (stQueue.size() > maxSize) { st = stQueue.poll(); @@ -120,6 +128,7 @@ maxBoostAtt.setCompetitiveTerm(t.bytes); } } + return true; } }); @@ -131,7 +140,7 @@ for (final ScoreTerm st : scoreTerms) { final Term term = placeholderTerm.createTerm(st.bytes); assert reader.docFreq(term) == st.docFreq; - addClause(q, term, st.docFreq, query.getBoost() * st.boost); // add to query + addClause(q, term, st.docFreq, query.getBoost() * st.boost, st.termState); // add to query } query.incTotalNumberOfTerms(scoreTerms.length); return q; @@ -167,7 +176,7 @@ public final BytesRef bytes = new BytesRef(); public float boost; public int docFreq; - + public final PerReaderTermState termState = new PerReaderTermState(); public ScoreTerm(Comparator termComp) { this.termComp = termComp; } Index: lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java =================================================================== --- lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (working copy) @@ -24,6 +24,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; @@ -285,11 +286,6 @@ } @Override - public void cacheCurrentTerm() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override public BytesRef term() throws IOException { return term; } @@ -318,6 +314,23 @@ public Comparator getComparator() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TermState getTermState() throws IOException { + throw new UnsupportedOperationException(); + } } } } Index: lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopTermsRewrite; @@ -153,7 +154,7 @@ } @Override - protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost) { + protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) { final SpanTermQuery q = new SpanTermQuery(term); q.setBoost(boost); topLevel.addClause(q); @@ -202,7 +203,7 @@ } @Override - protected void addClause(SpanOrQuery topLevel, Term term, int docFreq, float boost) { + protected void addClause(SpanOrQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) { final SpanTermQuery q = new SpanTermQuery(term); q.setBoost(boost); topLevel.addClause(q); Index: lucene/src/java/org/apache/lucene/util/ArrayUtil.java =================================================================== --- lucene/src/java/org/apache/lucene/util/ArrayUtil.java (revision 1036841) +++ lucene/src/java/org/apache/lucene/util/ArrayUtil.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import java.lang.reflect.Array; import java.util.Collection; import java.util.Comparator; @@ -365,6 +366,16 @@ } else return array; } + + public static T[] grow(Class clazz, T[] array, int minSize) { + if (array.length < minSize) { + @SuppressWarnings("unchecked") + T[] newArray = (T[]) Array.newInstance(clazz, oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJ_REF)); + System.arraycopy(array, 0, newArray, 0, array.length); + return newArray; + } else + return array; + } public static char[] grow(char[] array) { return grow(array, 1 + array.length); Index: lucene/src/test/org/apache/lucene/TestExternalCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/TestExternalCodecs.java (revision 1036841) +++ lucene/src/test/org/apache/lucene/TestExternalCodecs.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.util.*; +import org.apache.lucene.util.Bits; import org.apache.lucene.index.*; import org.apache.lucene.document.*; import org.apache.lucene.search.*; @@ -330,10 +331,6 @@ } @Override - public void cacheCurrentTerm() { - } - - @Override public DocsEnum docs(Bits skipDocs, DocsEnum reuse) { return new RAMDocsEnum(ramField.termToDocs.get(current), skipDocs); } @@ -342,6 +339,23 @@ public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) { return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), skipDocs); } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, TermState termState) + throws IOException { + return docs(skipDocs, reuse); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, + DocsAndPositionsEnum reuse, TermState termState) throws IOException { + return docsAndPositions(skipDocs, reuse); + } + + @Override + public TermState getTermState() throws IOException { + return null; + } } private static class RAMDocsEnum extends DocsEnum { @@ -580,7 +594,6 @@ pulsingReader, state.readBufferSize, reverseUnicodeComparator, - StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; return ret; Index: lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (revision 1036841) +++ lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (working copy) @@ -165,7 +165,6 @@ postingsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; return ret; Index: lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (revision 1036841) +++ lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (working copy) @@ -189,7 +189,6 @@ postingsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; return ret; Index: lucene/src/test/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (revision 1036841) +++ lucene/src/test/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (working copy) @@ -115,7 +115,6 @@ postingsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; return ret;