Index: lucene/src/java/org/apache/lucene/index/codecs/PerReaderTermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PerReaderTermState.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/PerReaderTermState.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.IdentityHashMap; +import java.util.Map; + +import org.apache.lucene.index.IndexReader; + +/** + * + * @lucene.experimental + */ +public class PerReaderTermState { + + + private final Map mapping = new IdentityHashMap(); + + public void set(TermState state, IndexReader reader) { + mapping.put(reader, state); + } + + public TermState get(IndexReader reader) { + return mapping.get(reader); + } +} Index: lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (working copy) @@ -42,17 +42,17 @@ public abstract void init(IndexInput termsIn) throws IOException; /** Return a newly created empty TermState */ - public abstract TermState newTermState() throws IOException; + public abstract StandardTermState newTermState() throws IOException; - public abstract void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState state, boolean isIndexTerm) throws IOException; + public abstract void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState state, boolean isIndexTerm) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsEnum docs(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsEnum reuse) throws IOException; + public abstract DocsEnum docs(FieldInfo fieldInfo, StandardTermState state, Bits skipDocs, DocsEnum reuse) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; + public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, StandardTermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; public abstract void close() throws IOException; } Index: lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (working copy) @@ -68,7 +68,7 @@ private final Comparator termComp; // Caches the most recently looked-up field + terms: - private final DoubleBarrelLRUCache termsCache; + private final DoubleBarrelLRUCache termsCache; // Reads the terms index private TermsIndexReaderBase indexReader; @@ -116,7 +116,7 @@ throws IOException { this.postingsReader = postingsReader; - termsCache = new DoubleBarrelLRUCache(termsCacheSize); + termsCache = new DoubleBarrelLRUCache(termsCacheSize); this.termComp = termComp; @@ -285,18 +285,22 @@ private class SegmentTermsEnum extends TermsEnum { private final IndexInput in; private final DeltaBytesReader bytesReader; - private final TermState state; + private final StandardTermState state; private boolean seekPending; private final TermsIndexReaderBase.TermsIndexResult indexResult = new TermsIndexReaderBase.TermsIndexResult(); private final FieldAndTerm fieldTerm = new FieldAndTerm(); - + SegmentTermsEnum() throws IOException { in = (IndexInput) PrefixCodedTermsReader.this.in.clone(); in.seek(termsStartPointer); bytesReader = new DeltaBytesReader(in); fieldTerm.field = fieldInfo.name; state = postingsReader.newTermState(); + state.info = fieldInfo; state.ord = -1; + final TermStateAttribute stateAttr = this.attributes().addAttribute(TermStateAttribute.class); + stateAttr.set(state); + } @Override @@ -306,7 +310,7 @@ @Override public void cacheCurrentTerm() { - TermState stateCopy = (TermState) state.clone(); + StandardTermState stateCopy = (StandardTermState) state.clone(); stateCopy.filePointer = in.getFilePointer(); termsCache.put(new FieldAndTerm(fieldInfo.name, bytesReader.term), stateCopy); @@ -320,7 +324,7 @@ public SeekStatus seek(BytesRef term, boolean useCache) throws IOException { // Check cache fieldTerm.term = term; - TermState cachedState; + StandardTermState cachedState; if (useCache) { cachedState = termsCache.get(fieldTerm); if (cachedState != null) { @@ -391,7 +395,7 @@ if (doSeek && useCache) { // Store in cache FieldAndTerm entryKey = new FieldAndTerm(fieldTerm); - cachedState = (TermState) state.clone(); + cachedState = (StandardTermState) state.clone(); // this is fp after current term cachedState.filePointer = in.getFilePointer(); termsCache.put(entryKey, cachedState); Index: lucene/src/java/org/apache/lucene/index/codecs/StandardTermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/StandardTermState.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/index/codecs/StandardTermState.java (working copy) @@ -17,9 +17,14 @@ * limitations under the License. */ +import java.io.IOException; + +import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; // for javadocs +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs +import org.apache.lucene.util.Bits; /** * Holds all state required for {@link StandardPostingsReader} @@ -28,29 +33,46 @@ * @lucene.experimental */ -public class TermState implements Cloneable { +public class StandardTermState extends TermState { public long ord; // ord for this term public long filePointer; // fp into the terms dict primary file (_X.tis) public int docFreq; // how many docs have this term - + + private final PostingsReaderBase reader; + public FieldInfo info; + + public StandardTermState(PostingsReaderBase reader) { + this.reader = reader; + } public void copy(TermState other) { - ord = other.ord; - filePointer = other.filePointer; - docFreq = other.docFreq; + StandardTermState state = (StandardTermState)other; + assert reader == state.reader; + ord = state.ord; + filePointer = state.filePointer; + docFreq = state.docFreq; + info = state.info; } @Override public Object clone() { - try { - return super.clone(); - } catch (CloneNotSupportedException cnse) { - // should not happen - throw new RuntimeException(cnse); - } + StandardTermState clone = (StandardTermState) super.clone(); + clone.info = info; + clone.ord = ord; + clone.filePointer = filePointer; + clone.docFreq = docFreq; + return clone; } @Override public String toString() { return "tis.fp=" + filePointer + " docFreq=" + docFreq + " ord=" + ord; } + @Override + public DocsAndPositionsEnum termPositions(Bits skipDocs) throws IOException { + return reader.docsAndPositions(info, this, skipDocs, null); + } + @Override + public DocsEnum termDocsEnum(Bits skipDocs) throws IOException { + return reader.docs(info, this, skipDocs, null); + } } Index: lucene/src/java/org/apache/lucene/index/codecs/TermState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/TermState.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/index/codecs/TermState.java (working copy) @@ -1,5 +1,11 @@ package org.apache.lucene.index.codecs; +import java.io.IOException; + +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.Bits; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -17,40 +23,21 @@ * limitations under the License. */ -import org.apache.lucene.index.DocsEnum; // for javadocs - -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs +public abstract class TermState implements Cloneable { -/** - * Holds all state required for {@link StandardPostingsReader} - * to produce a {@link DocsEnum} without re-seeking the - * terms dict. - * @lucene.experimental - */ - -public class TermState implements Cloneable { - public long ord; // ord for this term - public long filePointer; // fp into the terms dict primary file (_X.tis) - public int docFreq; // how many docs have this term - - public void copy(TermState other) { - ord = other.ord; - filePointer = other.filePointer; - docFreq = other.docFreq; - } - - @Override + public abstract void copy(TermState other); + + public abstract DocsAndPositionsEnum termPositions(Bits skipDocs) throws IOException; + + public abstract DocsEnum termDocsEnum(Bits skipDoc) throws IOException; + public Object clone() { try { return super.clone(); - } catch (CloneNotSupportedException cnse) { + } catch (CloneNotSupportedException e) { // should not happen - throw new RuntimeException(cnse); - } + throw new RuntimeException(e); + } } - - @Override - public String toString() { - return "tis.fp=" + filePointer + " docFreq=" + docFreq + " ord=" + ord; - } + } Index: lucene/src/java/org/apache/lucene/index/codecs/TermStateAttribute.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/TermStateAttribute.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/TermStateAttribute.java (revision 0) @@ -0,0 +1,33 @@ +package org.apache.lucene.index.codecs; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.apache.lucene.util.Attribute; + +/** + * + */ +public interface TermStateAttribute extends Attribute { + + /** + * + * @return + */ + public abstract TermState get(); + + public abstract void set(TermState state); + +} Index: lucene/src/java/org/apache/lucene/index/codecs/TermStateAttributeImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/TermStateAttributeImpl.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/TermStateAttributeImpl.java (revision 0) @@ -0,0 +1,73 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.util.SetOnce; + +public class TermStateAttributeImpl extends AttributeImpl implements + TermStateAttribute { + + private SetOnce state = new SetOnce(); + + public TermState get() { + return state.get(); + } + + public void set(TermState state) { + this.state.set(state); + } + + @Override + public void clear() { + state = new SetOnce(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((state.get() == null) ? 0 : state.get().hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + TermStateAttributeImpl other = (TermStateAttributeImpl) obj; + if (state.get() == null) { + if (other.state.get() != null) + return false; + } else if (!state.get().equals(other.state.get())) + return false; + return true; + } + + @Override + public void copyTo(AttributeImpl target) { + if (target instanceof TermStateAttributeImpl) { + ((TermStateAttributeImpl) target).state.set(state.get()); + } + } + +} Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (working copy) @@ -22,8 +22,9 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.codecs.TermState; +import org.apache.lucene.index.codecs.StandardTermState; import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.TermState; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl.Document; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl.Position; import org.apache.lucene.store.IndexInput; @@ -57,11 +58,16 @@ wrappedPostingsReader.init(termsIn); } - private static class PulsingTermState extends TermState { + private static class PulsingTermState extends StandardTermState { private Document docs[]; - private TermState wrappedTermState; + private StandardTermState wrappedTermState; private boolean pendingIndexTerm; + public PulsingTermState(PostingsReaderBase reader) { + super(reader); + } + + @Override public Object clone() { PulsingTermState clone; clone = (PulsingTermState) super.clone(); @@ -72,10 +78,11 @@ clone.docs[i] = (Document) doc.clone(); } } - clone.wrappedTermState = (TermState) wrappedTermState.clone(); + clone.wrappedTermState = (StandardTermState) wrappedTermState.clone(); return clone; } + @Override public void copy(TermState _other) { super.copy(_other); PulsingTermState other = (PulsingTermState) _other; @@ -90,16 +97,15 @@ } @Override - public TermState newTermState() throws IOException { - PulsingTermState state = new PulsingTermState(); + public StandardTermState newTermState() throws IOException { + PulsingTermState state = new PulsingTermState(this); state.wrappedTermState = wrappedPostingsReader.newTermState(); state.docs = new Document[maxPulsingDocFreq]; return state; } @Override - public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState _termState, boolean isIndexTerm) throws IOException { - + public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState _termState, boolean isIndexTerm) throws IOException { PulsingTermState termState = (PulsingTermState) _termState; termState.pendingIndexTerm |= isIndexTerm; @@ -182,7 +188,7 @@ // TODO: we could actually reuse, by having TL that // holds the last wrapped reuse, and vice-versa @Override - public DocsEnum docs(FieldInfo field, TermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo field, StandardTermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { PulsingTermState termState = (PulsingTermState) _termState; if (termState.docFreq <= maxPulsingDocFreq) { if (reuse instanceof PulsingDocsEnum) { @@ -202,7 +208,7 @@ // TODO: -- not great that we can't always reuse @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo field, TermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo field, StandardTermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { PulsingTermState termState = (PulsingTermState) _termState; if (termState.docFreq <= maxPulsingDocFreq) { if (reuse instanceof PulsingDocsAndPositionsEnum) { Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.StandardTermState; import org.apache.lucene.index.codecs.TermState; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -129,18 +130,24 @@ } } - private static class SepTermState extends TermState { + private static class SepTermState extends StandardTermState { // We store only the seek point to the docs file because // the rest of the info (freqIndex, posIndex, etc.) is // stored in the docs file: IntIndexInput.Index docIndex; + + public SepTermState(PostingsReaderBase reader) { + super(reader); + } + @Override public Object clone() { SepTermState other = (SepTermState) super.clone(); other.docIndex = (IntIndexInput.Index) docIndex.clone(); return other; } + @Override public void copy(TermState _other) { super.copy(_other); SepTermState other = (SepTermState) _other; @@ -154,19 +161,19 @@ } @Override - public TermState newTermState() throws IOException { - final SepTermState state = new SepTermState(); + public StandardTermState newTermState() throws IOException { + final SepTermState state = new SepTermState(this); state.docIndex = docIn.index(); return state; } @Override - public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState termState, boolean isIndexTerm) throws IOException { + public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState termState, boolean isIndexTerm) throws IOException { ((SepTermState) termState).docIndex.read(termsIn, isIndexTerm); } @Override - public DocsEnum docs(FieldInfo fieldInfo, TermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, StandardTermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { final SepTermState termState = (SepTermState) _termState; SepDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SepDocsEnum)) { @@ -185,7 +192,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, StandardTermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { assert !fieldInfo.omitTermFreqAndPositions; final SepTermState termState = (SepTermState) _termState; SepDocsAndPositionsEnum postingsEnum; Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (working copy) @@ -27,6 +27,7 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.StandardTermState; import org.apache.lucene.index.codecs.TermState; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; @@ -82,11 +83,16 @@ maxSkipLevels = termsIn.readInt(); } - private static class DocTermState extends TermState { + private static class DocTermState extends StandardTermState { long freqOffset; long proxOffset; int skipOffset; - + + public DocTermState(PostingsReaderBase reader) { + super(reader); + } + + @Override public Object clone() { DocTermState other = (DocTermState) super.clone(); other.freqOffset = freqOffset; @@ -95,6 +101,7 @@ return other; } + @Override public void copy(TermState _other) { super.copy(_other); DocTermState other = (DocTermState) _other; @@ -109,8 +116,8 @@ } @Override - public TermState newTermState() { - return new DocTermState(); + public StandardTermState newTermState() { + return new DocTermState(this); } @Override @@ -127,9 +134,8 @@ } @Override - public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState termState, boolean isIndexTerm) + public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, StandardTermState termState, boolean isIndexTerm) throws IOException { - final DocTermState docTermState = (DocTermState) termState; if (isIndexTerm) { @@ -154,7 +160,7 @@ } @Override - public DocsEnum docs(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs, DocsEnum reuse) throws IOException { SegmentDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SegmentDocsEnum)) { docsEnum = new SegmentDocsEnum(freqIn); @@ -171,7 +177,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { if (fieldInfo.omitTermFreqAndPositions) { return null; } Index: lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; @@ -71,8 +72,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/) { - topLevel.add(new TermQuery(term, docFreq), BooleanClause.Occur.SHOULD); + protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/, PerReaderTermState states) { + topLevel.add(new TermQuery(term, docFreq, states), BooleanClause.Occur.SHOULD); } @Override Index: lucene/src/java/org/apache/lucene/search/MultiTermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.util.AttributeSource; @@ -158,8 +159,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost) { - final TermQuery tq = new TermQuery(term, docCount); + protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) { + final TermQuery tq = new TermQuery(term, docCount, states); tq.setBoost(boost); topLevel.add(tq, BooleanClause.Occur.SHOULD); } @@ -199,8 +200,8 @@ } @Override - protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost) { - final Query q = new ConstantScoreQuery(new QueryWrapperFilter(new TermQuery(term, docFreq))); + protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) { + final Query q = new ConstantScoreQuery(new QueryWrapperFilter(new TermQuery(term, docFreq, states))); q.setBoost(boost); topLevel.add(q, BooleanClause.Occur.SHOULD); } Index: lucene/src/java/org/apache/lucene/search/ScoringRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (working copy) @@ -29,6 +29,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; +import org.apache.lucene.index.codecs.TermState; +import org.apache.lucene.index.codecs.TermStateAttribute; import org.apache.lucene.search.MultiTermQuery.RewriteMethod; import org.apache.lucene.util.ArrayUtil; @@ -63,12 +66,18 @@ @Override protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost) { - final TermQuery tq = new TermQuery(term, docCount); - tq.setBoost(boost); - topLevel.add(tq, BooleanClause.Occur.SHOULD); + addClause(topLevel, term, docCount, boost, null); } @Override + protected void addClause(BooleanQuery topLevel, Term term, int docCount, + float boost, PerReaderTermState states) { + final TermQuery tq = new TermQuery(term, docCount, states); + tq.setBoost(boost); + topLevel.add(tq, BooleanClause.Occur.SHOULD); + } + + @Override protected void checkMaxClauseCount(int count) { if (count > BooleanQuery.getMaxClauseCount()) throw new BooleanQuery.TooManyClauses(); @@ -77,7 +86,7 @@ // Make sure we are still a singleton even after deserializing protected Object readResolve() { return SCORING_BOOLEAN_QUERY_REWRITE; - } + } }; /** Like {@link #SCORING_BOOLEAN_QUERY_REWRITE} except @@ -125,11 +134,12 @@ final int sort[] = col.terms.sort(col.termsEnum.getComparator()); final int[] docFreq = col.array.docFreq; final float[] boost = col.array.boost; + final PerReaderTermState[] termStates = col.array.termState; for (int i = 0; i < size; i++) { final int pos = sort[i]; final Term term = placeholderTerm.createTerm(col.terms.get(pos, new BytesRef())); assert reader.docFreq(term) == docFreq[pos]; - addClause(result, term, docFreq[pos], query.getBoost() * boost[pos]); + addClause(result, term, docFreq[pos], query.getBoost() * boost[pos], termStates[pos]); } } query.incTotalNumberOfTerms(size); @@ -142,25 +152,37 @@ TermsEnum termsEnum; private BoostAttribute boostAtt; + private TermStateAttribute termStateAttr; @Override public void setNextEnum(TermsEnum termsEnum) throws IOException { this.termsEnum = termsEnum; this.boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class); + this.termStateAttr = termsEnum.attributes().addAttribute(TermStateAttribute.class); } @Override public boolean collect(BytesRef bytes) throws IOException { final int e = terms.add(bytes); + TermState state; + if((state = termStateAttr.get()) != null) { + state = (TermState) state.clone(); + } if (e < 0 ) { // duplicate term: update docFreq final int pos = (-e)-1; array.docFreq[pos] += termsEnum.docFreq(); + if(state != null) + array.termState[pos].set(state, reader); assert array.boost[pos] == boostAtt.getBoost() : "boost should be equal in all segment TermsEnums"; } else { // new entry: we populate the entry initially array.docFreq[e] = termsEnum.docFreq(); array.boost[e] = boostAtt.getBoost(); + if(state != null) { + array.termState[e] = new PerReaderTermState(); + array.termState[e].set(state, reader); + } ScoringRewrite.this.checkMaxClauseCount(terms.size()); } return true; @@ -171,6 +193,7 @@ static final class TermFreqBoostByteStart extends DirectBytesStartArray { int[] docFreq; float[] boost; + PerReaderTermState[] termState; public TermFreqBoostByteStart(int initSize) { super(initSize); @@ -181,6 +204,8 @@ final int[] ord = super.init(); boost = new float[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_FLOAT)]; docFreq = new int[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_INT)]; + termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJ_REF)]; + assert boost.length >= ord.length && docFreq.length >= ord.length; return ord; } @@ -190,6 +215,7 @@ final int[] ord = super.grow(); docFreq = ArrayUtil.grow(docFreq, ord.length); boost = ArrayUtil.grow(boost, ord.length); + termState = ArrayUtil.grow(PerReaderTermState.class, termState, ord.length); assert boost.length >= ord.length && docFreq.length >= ord.length; return ord; } Index: lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java (working copy) @@ -28,6 +28,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ReaderUtil; @@ -38,7 +39,12 @@ protected abstract Q getTopLevelQuery() throws IOException; /** Add a MultiTermQuery term to the top-level query */ - protected abstract void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException; + protected void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException { + addClause(topLevel, term, docCount, boost, null); + } + + protected abstract void addClause(Q topLevel, Term term, int docCount, float boost, PerReaderTermState states) throws IOException; + protected final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException { final List subReaders = new ArrayList(); @@ -71,9 +77,10 @@ lastTermComp = newTermComp; collector.setNextEnum(termsEnum); + collector.setNextReader(r); BytesRef bytes; while ((bytes = termsEnum.next()) != null) { - termsEnum.cacheCurrentTerm(); +// termsEnum.cacheCurrentTerm(); if (!collector.collect(bytes)) return; // interrupt whole term collection, so also don't iterate other subReaders } @@ -81,6 +88,8 @@ } protected static abstract class TermCollector { + protected IndexReader reader; + /** attributes used for communication with the enum */ public final AttributeSource attributes = new AttributeSource(); @@ -89,5 +98,9 @@ /** the next segment's {@link TermsEnum} that is used to collect terms */ public abstract void setNextEnum(TermsEnum termsEnum) throws IOException; + + public void setNextReader(IndexReader reader) { + this.reader = reader; + } } } Index: lucene/src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermQuery.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -23,6 +23,8 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.codecs.PerReaderTermState; +import org.apache.lucene.index.codecs.TermState; import org.apache.lucene.search.Explanation.IDFExplanation; import org.apache.lucene.util.ToStringUtils; @@ -32,6 +34,7 @@ public class TermQuery extends Query { private final Term term; private final int docFreq; + private final PerReaderTermState perReaderTermState; private class TermWeight extends Weight { private final Similarity similarity; @@ -76,10 +79,15 @@ @Override public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), - term.field(), - term.bytes()); - + TermState state; + final DocsEnum docs; + if(perReaderTermState != null && (state = perReaderTermState.get(reader)) != null) { + docs = state.termDocsEnum(reader.getDeletedDocs()); + } else { + docs = reader.termDocsEnum(reader.getDeletedDocs(), + term.field(), + term.bytes()); + } if (docs == null) { return null; } @@ -174,6 +182,16 @@ public TermQuery(Term t, int docFreq) { term = t; this.docFreq = docFreq; + this.perReaderTermState = null; + } + + /** Expert: constructs a TermQuery that will use the + * provided docFreq instead of looking up the docFreq + * against the searcher. */ + public TermQuery(Term t, int docFreq, PerReaderTermState states) { + term = t; + this.docFreq = docFreq; + this.perReaderTermState = states; } /** Returns the term of this query. */ Index: lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.codecs.PerReaderTermState; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopTermsRewrite; @@ -153,7 +154,7 @@ } @Override - protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost) { + protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) { final SpanTermQuery q = new SpanTermQuery(term); q.setBoost(boost); topLevel.addClause(q); @@ -202,7 +203,7 @@ } @Override - protected void addClause(SpanOrQuery topLevel, Term term, int docFreq, float boost) { + protected void addClause(SpanOrQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) { final SpanTermQuery q = new SpanTermQuery(term); q.setBoost(boost); topLevel.addClause(q); Index: lucene/src/java/org/apache/lucene/util/ArrayUtil.java =================================================================== --- lucene/src/java/org/apache/lucene/util/ArrayUtil.java (revision 1036343) +++ lucene/src/java/org/apache/lucene/util/ArrayUtil.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import java.lang.reflect.Array; import java.util.Collection; import java.util.Comparator; @@ -365,6 +366,16 @@ } else return array; } + + public static T[] grow(Class clazz, T[] array, int minSize) { + if (array.length < minSize) { + @SuppressWarnings("unchecked") + T[] newArray = (T[]) Array.newInstance(clazz, oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJ_REF)); + System.arraycopy(array, 0, newArray, 0, array.length); + return newArray; + } else + return array; + } public static char[] grow(char[] array) { return grow(array, 1 + array.length);