Index: org/apache/lucene/document/AbstractField.java =================================================================== --- org/apache/lucene/document/AbstractField.java (revision 653443) +++ org/apache/lucene/document/AbstractField.java (working copy) @@ -32,6 +32,7 @@ protected boolean isTokenized = true; protected boolean isBinary = false; protected boolean isCompressed = false; + protected boolean isStoreTermDocs = false; protected boolean lazy = false; protected float boost = 1.0f; // the one and only data object for all different kind of field values @@ -42,7 +43,7 @@ } - protected AbstractField(String name, Field.Store store, Field.Index index, Field.TermVector termVector) { + protected AbstractField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, Field.TermDocs termDocs) { if (name == null) throw new NullPointerException("name cannot be null"); this.name = name.intern(); // field names are interned @@ -78,7 +79,15 @@ } else { throw new IllegalArgumentException("unknown index parameter " + index); } - + + if (termDocs == Field.TermDocs.STORE) { + isStoreTermDocs = true; + } else if (termDocs == Field.TermDocs.NO) { + isStoreTermDocs = false; + } else { + throw new IllegalArgumentException("unknown index parameter " + index); + } + this.isBinary = false; setStoreTermVector(termVector); @@ -159,7 +168,9 @@ with search hits. It is an error for this to be true if a field is Reader-valued. */ public final boolean isStored() { return isStored; } - + + public final boolean isStoreTermDocs() { return isStoreTermDocs; } + /** True iff the value of the field is to be indexed, so that it may be searched on. */ public final boolean isIndexed() { return isIndexed; } Index: org/apache/lucene/document/Field.java =================================================================== --- org/apache/lucene/document/Field.java (revision 653443) +++ org/apache/lucene/document/Field.java (working copy) @@ -56,14 +56,31 @@ /** Do not store the field value in the index. */ public static final Store NO = new Store("NO"); } - + /** Specifies whether and how a field should be indexed. */ + public static final class TermDocs extends Parameter implements Serializable { + private TermDocs(String name) { + super(name); + } + + /** + * Store docs with the term in the term dictionary + */ + public static final TermDocs STORE = new TermDocs("STORE"); + + /** + * Do not store docs with the term in the term dictionary + */ + public static final TermDocs NO = new TermDocs("NO"); + } + + /** Specifies whether and how a field should be indexed. */ public static final class Index extends Parameter implements Serializable { private Index(String name) { super(name); } - + /** Do not index the field value. This field can thus not be searched, * but one can still access its contents provided it is * {@link Field.Store stored}. */ @@ -202,9 +219,17 @@ * @throws IllegalArgumentException if the field is neither stored nor indexed */ public Field(String name, String value, Store store, Index index) { - this(name, value, store, index, TermVector.NO); + this(name, value, store, index, TermVector.NO, TermDocs.NO); } + public Field(String name, String value, Store store, Index index, TermDocs termDocs) { + this(name, value, store, index, TermVector.NO, termDocs); + } + + public Field(String name, String value, Store store, Index index, TermVector termVector) { + this(name, value, store, index, termVector, TermDocs.NO); + } + /** * Create a field by specifying its name, value and how it will * be saved in the index. @@ -222,7 +247,7 @@ *
  • the field is not indexed but termVector is TermVector.YES
    • * */ - public Field(String name, String value, Store store, Index index, TermVector termVector) { + public Field(String name, String value, Store store, Index index, TermVector termVector, TermDocs termDocs) { if (name == null) throw new NullPointerException("name cannot be null"); if (value == null) @@ -271,6 +296,14 @@ throw new IllegalArgumentException("unknown index parameter " + index); } + if (termDocs == TermDocs.NO) { + this.isStoreTermDocs = false; + } else if (termDocs == TermDocs.STORE) { + this.isStoreTermDocs = true; + } else { + throw new IllegalArgumentException("unknown termdocs parameter " + termDocs); + } + this.isBinary = false; setStoreTermVector(termVector); Index: org/apache/lucene/document/Fieldable.java =================================================================== --- org/apache/lucene/document/Fieldable.java (revision 653443) +++ org/apache/lucene/document/Fieldable.java (working copy) @@ -108,7 +108,9 @@ * @see org.apache.lucene.index.IndexReader#getTermFreqVector(int, String) */ boolean isTermVectorStored(); - + + boolean isStoreTermDocs(); + /** * True iff terms are stored as term vector together with their offsets * (start and end positon in source text). Index: org/apache/lucene/index/DocumentsWriter.java =================================================================== --- org/apache/lucene/index/DocumentsWriter.java (revision 653443) +++ org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -17,30 +17,31 @@ * limitations under the License. */ +import java.io.IOException; +import java.io.PrintStream; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Similarity; import org.apache.lucene.search.Weight; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.util.IntArrayList; import org.apache.lucene.util.UnicodeUtil; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.HashMap; -import java.util.ArrayList; -import java.util.Map.Entry; -import java.text.NumberFormat; -import java.util.Collections; - /** * This class accepts multiple added documents and directly * writes a single segment file. It does this more @@ -181,7 +182,10 @@ private int maxBufferedDocs = IndexWriter.DEFAULT_MAX_BUFFERED_DOCS; private int flushedDocCount; // How many docs already flushed to index - + + //Used for append posting term docs + private IntArrayList termDocsList = new IntArrayList(50, 1.25); + synchronized void updateFlushedDocCount(int n) { flushedDocCount += n; } @@ -805,7 +809,6 @@ currentFieldStorePayloads = fields[0].fieldInfo.storePayloads; DocumentsWriterFieldMergeState[] termStates = new DocumentsWriterFieldMergeState[numFields]; - while(numFields > 0) { // Get the next term to merge @@ -841,7 +844,8 @@ // which all share the same term. Now we must // interleave the docID streams. while(numToMerge > 0) { - + boolean doTermDocs = termStates[0].field.doTermDocs; + termDocsList.clear(); if ((++df % skipInterval) == 0) { skipListWriter.setSkipData(lastDoc, currentFieldStorePayloads, lastPayloadLength); skipListWriter.bufferSkip(df); @@ -854,7 +858,9 @@ final int doc = minState.docID; final int termDocFreq = minState.termFreq; - + + if (doTermDocs) termDocsList.add(doc); + assert doc < numDocsInRAM; assert doc > lastDoc || df == 1; @@ -926,9 +932,15 @@ // Done merging this term long skipPointer = skipListWriter.writeSkip(freqOut); - + + // create the term docs array if needed + int[] docs = null; + if (termStates[0].field.doTermDocs) { + docs = termDocsList.toIntArray(); + } + // Write term - termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer)); + termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer), docs); // TODO: we could do this incrementally UnicodeUtil.UTF16toUTF8(text, start, termsUTF8); Index: org/apache/lucene/index/DocumentsWriterFieldData.java =================================================================== --- org/apache/lucene/index/DocumentsWriterFieldData.java (revision 653443) +++ org/apache/lucene/index/DocumentsWriterFieldData.java (working copy) @@ -50,6 +50,7 @@ boolean doVectorPositions; boolean doVectorOffsets; boolean postingsCompacted; + boolean doTermDocs; int numPostings; Index: org/apache/lucene/index/DocumentsWriterThreadState.java =================================================================== --- org/apache/lucene/index/DocumentsWriterThreadState.java (revision 653443) +++ org/apache/lucene/index/DocumentsWriterThreadState.java (working copy) @@ -206,7 +206,7 @@ FieldInfo fi = docWriter.fieldInfos.add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), - field.getOmitNorms(), false); + field.getOmitNorms(), false, field.isStoreTermDocs()); if (fi.isIndexed && !fi.omitNorms) { // Maybe grow our buffered norms if (docWriter.norms.length <= fi.number) { @@ -270,6 +270,7 @@ fp.fieldCount = 0; fp.doVectors = fp.doVectorPositions = fp.doVectorOffsets = false; fp.doNorms = fi.isIndexed && !fi.omitNorms; + fp.doTermDocs = fi.storeTermDoc; if (numFieldData == fieldDataArray.length) { int newSize = fieldDataArray.length*2; Index: org/apache/lucene/index/FieldInfo.java =================================================================== --- org/apache/lucene/index/FieldInfo.java (revision 653443) +++ org/apache/lucene/index/FieldInfo.java (working copy) @@ -26,6 +26,7 @@ boolean storeTermVector; boolean storeOffsetWithTermVector; boolean storePositionWithTermVector; + boolean storeTermDoc; boolean omitNorms; // omit norms associated with indexed fields @@ -33,7 +34,7 @@ FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, - boolean omitNorms, boolean storePayloads) { + boolean omitNorms, boolean storePayloads, boolean storeTermDoc) { name = na; isIndexed = tk; number = nu; @@ -42,10 +43,11 @@ this.storePositionWithTermVector = storePositionWithTermVector; this.omitNorms = omitNorms; this.storePayloads = storePayloads; + this.storeTermDoc = storeTermDoc; } public Object clone() { return new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, storePayloads); + storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDoc); } } Index: org/apache/lucene/index/FieldInfos.java =================================================================== --- org/apache/lucene/index/FieldInfos.java (revision 653443) +++ org/apache/lucene/index/FieldInfos.java (working copy) @@ -40,6 +40,7 @@ static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x8; static final byte OMIT_NORMS = 0x10; static final byte STORE_PAYLOADS = 0x20; + static final byte STORE_TERM_DOCS = 0x40; private ArrayList byNumber = new ArrayList(); private HashMap byName = new HashMap(); @@ -83,7 +84,7 @@ while (fieldIterator.hasNext()) { Fieldable field = (Fieldable) fieldIterator.next(); add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), - field.isStoreOffsetWithTermVector(), field.getOmitNorms()); + field.isStoreOffsetWithTermVector(), field.getOmitNorms(), field.isStoreTermDocs()); } } @@ -96,10 +97,10 @@ * @param storeOffsetWithTermVector true if offsets should be stored */ public void addIndexed(Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector) { + boolean storeOffsetWithTermVector, boolean storeTermDocs) { Iterator i = names.iterator(); while (i.hasNext()) { - add((String)i.next(), true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector); + add((String)i.next(), true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, storeTermDocs); } } @@ -126,7 +127,7 @@ * @see #add(String, boolean, boolean, boolean, boolean) */ public void add(String name, boolean isIndexed) { - add(name, isIndexed, false, false, false, false); + add(name, isIndexed, false, false, false, false, false); } /** @@ -137,7 +138,7 @@ * @param storeTermVector true if the term vector should be stored */ public void add(String name, boolean isIndexed, boolean storeTermVector){ - add(name, isIndexed, storeTermVector, false, false, false); + add(name, isIndexed, storeTermVector, false, false, false, false); } /** If the field is not yet known, adds it. If it is known, checks to make @@ -152,9 +153,9 @@ * @param storeOffsetWithTermVector true if the term vector with offsets should be stored */ public void add(String name, boolean isIndexed, boolean storeTermVector, - boolean storePositionWithTermVector, boolean storeOffsetWithTermVector) { + boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean storeTermDocs) { - add(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, false); + add(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, false, storeTermDocs); } /** If the field is not yet known, adds it. If it is known, checks to make @@ -170,9 +171,9 @@ * @param omitNorms true if the norms for the indexed field should be omitted */ public void add(String name, boolean isIndexed, boolean storeTermVector, - boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms) { + boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms, boolean storeTermDocs) { add(name, isIndexed, storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, false); + storeOffsetWithTermVector, omitNorms, false, storeTermDocs); } /** If the field is not yet known, adds it. If it is known, checks to make @@ -190,10 +191,10 @@ */ public FieldInfo add(String name, boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, - boolean omitNorms, boolean storePayloads) { + boolean omitNorms, boolean storePayloads, boolean storeTermDocs) { FieldInfo fi = fieldInfo(name); if (fi == null) { - return addInternal(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads); + return addInternal(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDocs); } else { if (fi.isIndexed != isIndexed) { fi.isIndexed = true; // once indexed, always index @@ -213,17 +214,19 @@ if (fi.storePayloads != storePayloads) { fi.storePayloads = true; } - + if (fi.storeTermDoc != storeTermDocs) { + fi.storeTermDoc = true; + } } return fi; } private FieldInfo addInternal(String name, boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads) { + boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean storeTermDocs) { FieldInfo fi = new FieldInfo(name, isIndexed, byNumber.size(), storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, storePayloads); + storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDocs); byNumber.add(fi); byName.put(name, fi); return fi; @@ -295,6 +298,7 @@ if (fi.storeOffsetWithTermVector) bits |= STORE_OFFSET_WITH_TERMVECTOR; if (fi.omitNorms) bits |= OMIT_NORMS; if (fi.storePayloads) bits |= STORE_PAYLOADS; + if (fi.storeTermDoc) bits |= STORE_TERM_DOCS; output.writeString(fi.name); output.writeByte(bits); } @@ -311,8 +315,8 @@ boolean storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0; boolean omitNorms = (bits & OMIT_NORMS) != 0; boolean storePayloads = (bits & STORE_PAYLOADS) != 0; - - addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads); + boolean storeTermDocs = (bits & STORE_TERM_DOCS) != 0; + addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDocs); } } Index: org/apache/lucene/index/FieldsReader.java =================================================================== --- org/apache/lucene/index/FieldsReader.java (revision 653443) +++ org/apache/lucene/index/FieldsReader.java (working copy) @@ -280,6 +280,7 @@ Field.Store store = Field.Store.YES; Field.Index index = getIndexType(fi, tokenize); Field.TermVector termVector = getTermVectorType(fi); + Field.TermDocs termDocs = getTermDocsType(fi); Fieldable f; if (compressed) { @@ -298,7 +299,7 @@ fieldsStream.seek(pointer+length); else fieldsStream.skipChars(length); - f = new LazyField(fi.name, store, index, termVector, length, pointer, binary); + f = new LazyField(fi.name, store, index, termVector, termDocs, length, pointer, binary); f.setOmitNorms(fi.omitNorms); } doc.add(f); @@ -397,7 +398,15 @@ } return termVector; } - + + private Field.TermDocs getTermDocsType(FieldInfo fi) { + if (fi.storeTermDoc) { + return Field.TermDocs.STORE; + } else { + return Field.TermDocs.NO; + } + } + private Field.Index getIndexType(FieldInfo fi, boolean tokenize) { Field.Index index; if (fi.isIndexed && tokenize) @@ -418,15 +427,15 @@ private long pointer; public LazyField(String name, Field.Store store, int toRead, long pointer, boolean isBinary) { - super(name, store, Field.Index.NO, Field.TermVector.NO); + super(name, store, Field.Index.NO, Field.TermVector.NO, Field.TermDocs.NO); this.toRead = toRead; this.pointer = pointer; this.isBinary = isBinary; lazy = true; } - public LazyField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, int toRead, long pointer, boolean isBinary) { - super(name, store, index, termVector); + public LazyField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, Field.TermDocs termDocs, int toRead, long pointer, boolean isBinary) { + super(name, store, index, termVector, termDocs); this.toRead = toRead; this.pointer = pointer; this.isBinary = isBinary; Index: org/apache/lucene/index/FilterIndexReader.java =================================================================== --- org/apache/lucene/index/FilterIndexReader.java (revision 653443) +++ org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -87,6 +87,7 @@ public boolean next() throws IOException { return in.next(); } public Term term() { return in.term(); } public int docFreq() { return in.docFreq(); } + public int[] docs() { return in.docs(); } public void close() throws IOException { in.close(); } } @@ -177,7 +178,17 @@ protected void doSetNorm(int d, String f, byte b) throws CorruptIndexException, IOException { in.setNorm(d, f, b); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return in.terms(loadDocs); + } + public TermEnum terms(Term t, boolean loadDocs) throws IOException { + ensureOpen(); + return in.terms(t, loadDocs); + } + public TermEnum terms() throws IOException { ensureOpen(); return in.terms(); Index: org/apache/lucene/index/IndexReader.java =================================================================== --- org/apache/lucene/index/IndexReader.java (revision 653443) +++ org/apache/lucene/index/IndexReader.java (working copy) @@ -89,6 +89,8 @@ public static final FieldOption TERMVECTOR_WITH_OFFSET = new FieldOption ("TERMVECTOR_WITH_OFFSET"); /** All fields with termvectors with offset values and position values enabled */ public static final FieldOption TERMVECTOR_WITH_POSITION_OFFSET = new FieldOption ("TERMVECTOR_WITH_POSITION_OFFSET"); + + public static final FieldOption TERM_DOCS = new FieldOption ("TERMVECTOR_DOCS"); } private boolean closed; @@ -645,6 +647,10 @@ * @throws IOException if there is a low-level IO error */ public abstract TermEnum terms(Term t) throws IOException; + + public abstract TermEnum terms(Term t, boolean loadDocs) throws IOException; + + public abstract TermEnum terms(boolean loadDocs) throws IOException; /** Returns the number of documents containing the term t. * @throws IOException if there is a low-level IO error Index: org/apache/lucene/index/MultiReader.java =================================================================== --- org/apache/lucene/index/MultiReader.java (revision 653443) +++ org/apache/lucene/index/MultiReader.java (working copy) @@ -295,14 +295,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, null); + return new MultiTermEnum(subReaders, starts, null, false); } public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, term); + return new MultiTermEnum(subReaders, starts, term, false); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, null, loadDocs); + } + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, term, loadDocs); + } + public int docFreq(Term t) throws IOException { ensureOpen(); int total = 0; // sum freqs in segments Index: org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- org/apache/lucene/index/MultiSegmentReader.java (revision 653443) +++ org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -17,10 +17,6 @@ * limitations under the License. */ -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.store.Directory; - import java.io.IOException; import java.util.Collection; import java.util.HashMap; @@ -30,6 +26,11 @@ import java.util.Map; import java.util.Set; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IntArrayList; + /** * An IndexReader which reads indexes with multiple segments. */ @@ -358,14 +359,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, null); + return new MultiTermEnum(subReaders, starts, null, false); } - + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, null, loadDocs); + } + public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, term); + return new MultiTermEnum(subReaders, starts, term, false); } - + + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, term, loadDocs); + } + public int docFreq(Term t) throws IOException { ensureOpen(); int total = 0; // sum freqs in segments @@ -449,18 +460,22 @@ private Term term; private int docFreq; + private boolean loadDocs; + private IntArrayList termDocsList = new IntArrayList(50, 1.25); + private int[] docs; - public MultiTermEnum(IndexReader[] readers, int[] starts, Term t) + public MultiTermEnum(IndexReader[] readers, int[] starts, Term t, boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; queue = new SegmentMergeQueue(readers.length); for (int i = 0; i < readers.length; i++) { IndexReader reader = readers[i]; TermEnum termEnum; if (t != null) { - termEnum = reader.terms(t); + termEnum = reader.terms(t, loadDocs); } else - termEnum = reader.terms(); + termEnum = reader.terms(loadDocs); SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader); if (t == null ? smi.next() : termEnum.term() != null) @@ -473,19 +488,29 @@ next(); } } - + public boolean next() throws IOException { SegmentMergeInfo top = (SegmentMergeInfo)queue.top(); if (top == null) { term = null; return false; } - + docs = null; term = top.term; docFreq = 0; - + termDocsList.clear(); while (top != null && term.compareTo(top.term) == 0) { queue.pop(); + docFreq += top.termEnum.docFreq(); // increment freq + // merge doc ids + if (loadDocs) { + int[] readerDocs = top.termEnum.docs(); + if (readerDocs != null) { + for (int x = 0; x < readerDocs.length; x++) { + termDocsList.add(top.base + readerDocs[x]); + } + } + } docFreq += top.termEnum.docFreq(); // increment freq if (top.next()) queue.put(top); // restore queue @@ -493,9 +518,14 @@ top.close(); // done with a segment top = (SegmentMergeInfo)queue.top(); } + docs = termDocsList.toIntArray(); return true; } - + + public int[] docs() { + return docs; + } + public Term term() { return term; } Index: org/apache/lucene/index/ParallelReader.java =================================================================== --- org/apache/lucene/index/ParallelReader.java (revision 653443) +++ org/apache/lucene/index/ParallelReader.java (working copy) @@ -350,14 +350,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new ParallelTermEnum(); + return new ParallelTermEnum(false); } public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new ParallelTermEnum(term); + return new ParallelTermEnum(term, false); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new ParallelTermEnum(loadDocs); + } + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new ParallelTermEnum(term, loadDocs); + } + public int docFreq(Term term) throws IOException { ensureOpen(); IndexReader reader = ((IndexReader)fieldToReader.get(term.field())); @@ -455,18 +465,21 @@ private String field; private Iterator fieldIterator; private TermEnum termEnum; + private boolean loadDocs; - public ParallelTermEnum() throws IOException { + public ParallelTermEnum(boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; field = (String)fieldToReader.firstKey(); if (field != null) - termEnum = ((IndexReader)fieldToReader.get(field)).terms(); + termEnum = ((IndexReader)fieldToReader.get(field)).terms(loadDocs); } - public ParallelTermEnum(Term term) throws IOException { + public ParallelTermEnum(Term term, boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; field = term.field(); IndexReader reader = ((IndexReader)fieldToReader.get(field)); if (reader!=null) - termEnum = reader.terms(term); + termEnum = reader.terms(term, loadDocs); } public boolean next() throws IOException { @@ -510,7 +523,12 @@ return termEnum.docFreq(); } - + + public int[] docs() { + if (termEnum == null) return null; + return termEnum.docs(); + } + public void close() throws IOException { if (termEnum!=null) termEnum.close(); Index: org/apache/lucene/index/SegmentMerger.java =================================================================== --- org/apache/lucene/index/SegmentMerger.java (revision 653443) +++ org/apache/lucene/index/SegmentMerger.java (working copy) @@ -17,16 +17,17 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; import java.util.Vector; -import java.util.Iterator; -import java.util.Collection; -import java.io.IOException; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IntArrayList; /** * The SegmentMerger class combines two or more Segments, represented by an IndexReader ({@link #add}, @@ -64,7 +65,9 @@ /** Maximum number of contiguous documents to bulk-copy when merging stored fields */ private final static int MAX_RAW_MERGE_DOCS = 4192; - + + private IntArrayList termDocsList = new IntArrayList(50, 1.25); + /** This ctor used only by test code. * * @param dir The Directory to merge the other segments into @@ -197,11 +200,11 @@ } private void addIndexed(IndexReader reader, FieldInfos fieldInfos, Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector, boolean storePayloads) throws IOException { + boolean storeOffsetWithTermVector, boolean storePayloads, boolean storeTermDocs) throws IOException { Iterator i = names.iterator(); while (i.hasNext()) { String field = (String)i.next(); - fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads); + fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads, storeTermDocs); } } @@ -264,15 +267,17 @@ SegmentReader segmentReader = (SegmentReader) reader; for (int j = 0; j < segmentReader.getFieldInfos().size(); j++) { FieldInfo fi = segmentReader.getFieldInfos().fieldInfo(j); - fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads); + fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads, fi.storeTermDoc); } } else { - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERM_DOCS), false, false, false, false, true); + fieldInfos.add(reader.getFieldNames(IndexReader.FieldOption.UNINDEXED), false); } } @@ -502,7 +507,7 @@ top = (SegmentMergeInfo) queue.top(); } - final int df = mergeTermInfo(match, matchSize); // add new TermInfo + final int df = mergeTermInfo(match, matchSize, term); // add new TermInfo if (checkAbort != null) checkAbort.work(df/3.0); @@ -528,24 +533,39 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int mergeTermInfo(SegmentMergeInfo[] smis, int n) + private final int mergeTermInfo(SegmentMergeInfo[] smis, int n, Term term) throws CorruptIndexException, IOException { long freqPointer = freqOutput.getFilePointer(); long proxPointer = proxOutput.getFilePointer(); + + Holder docsHolder = new Holder(null); + // TODO: can optimize to not generate doc array if no need + int df = appendPostings(smis, n, docsHolder); // append posting data - int df = appendPostings(smis, n); // append posting data - long skipPointer = skipListWriter.writeSkip(freqOutput); if (df > 0) { + int[] docs = null; + FieldInfo fieldInfo = fieldInfos.fieldInfo(term.field()); + if (fieldInfo.storeTermDoc) { + docs = (int[])docsHolder.object; + } // add an entry to the dictionary with pointers to prox and freq files - termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer)); + termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer), docs); termInfosWriter.add(smis[0].term, termInfo); } return df; } + public static class Holder { + public Object object; + + public Holder(Object object) { + this.object = object; + } + } + private byte[] payloadBuffer; private int[][] docMaps; int[][] getDocMaps() { @@ -566,13 +586,14 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int appendPostings(SegmentMergeInfo[] smis, int n) + private final int appendPostings(SegmentMergeInfo[] smis, int n, Holder docsHolder) throws CorruptIndexException, IOException { int lastDoc = 0; int df = 0; // number of docs w/ term skipListWriter.resetSkip(); boolean storePayloads = fieldInfos.fieldInfo(smis[0].term.field).storePayloads; int lastPayloadLength = -1; // ensures that we write the first length + termDocsList.clear(); for (int i = 0; i < n; i++) { SegmentMergeInfo smi = smis[i]; TermPositions postings = smi.getPositions(); @@ -589,7 +610,9 @@ if (doc < 0 || (df > 0 && doc <= lastDoc)) throw new CorruptIndexException("docs out of order (" + doc + " <= " + lastDoc + " )"); - + + termDocsList.add(doc); + df++; if ((df % skipInterval) == 0) { @@ -638,6 +661,7 @@ } } } + docsHolder.object = termDocsList.toIntArray(); return df; } Index: org/apache/lucene/index/SegmentReader.java =================================================================== --- org/apache/lucene/index/SegmentReader.java (revision 653443) +++ org/apache/lucene/index/SegmentReader.java (working copy) @@ -653,7 +653,17 @@ ensureOpen(); return tis.terms(t); } + + public TermEnum terms(boolean loadDocs) { + ensureOpen(); + return tis.terms(loadDocs); + } + public TermEnum terms(Term t, boolean loadDocs) throws IOException { + ensureOpen(); + return tis.terms(t, loadDocs); + } + FieldInfos getFieldInfos() { return fieldInfos; } Index: org/apache/lucene/index/SegmentTermEnum.java =================================================================== --- org/apache/lucene/index/SegmentTermEnum.java (revision 653443) +++ org/apache/lucene/index/SegmentTermEnum.java (working copy) @@ -18,7 +18,9 @@ */ import java.io.IOException; + import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.VIntUtil; final class SegmentTermEnum extends TermEnum implements Cloneable { private IndexInput input; @@ -39,12 +41,14 @@ int skipInterval; int maxSkipLevels; private int formatM1SkipInterval; + private boolean loadDocs = false; - SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi) + SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi, boolean loadDocs) throws CorruptIndexException, IOException { input = i; fieldInfos = fis; isIndex = isi; + this.loadDocs = loadDocs; maxSkipLevels = 1; // use single-level skip lists for formats > -3 int firstInt = input.readInt(); @@ -89,7 +93,11 @@ prevBuffer.setPreUTF8Strings(); } } - + + void setLoadDocs(boolean loadDocs) { + this.loadDocs = loadDocs; + } + protected Object clone() { SegmentTermEnum clone = null; try { @@ -125,7 +133,19 @@ prevBuffer.set(termBuffer); termBuffer.read(input, fieldInfos); - + + // Read docs + int bytesLength = input.readVInt(); + if (loadDocs && bytesLength >= 0) { + int numDocs = input.readVInt(); + byte[] bytes = new byte[bytesLength]; + input.readBytes(bytes, 0, bytesLength); + int[] docs = VIntUtil.toArray(numDocs, bytes); + termInfo.docs = docs; + } else { + input.seek(input.getFilePointer() + bytesLength + 1); + } + termInfo.docFreq = input.readVInt(); // read doc freq termInfo.freqPointer += input.readVLong(); // read freq pointer termInfo.proxPointer += input.readVLong(); // read prox pointer @@ -178,7 +198,14 @@ final void termInfo(TermInfo ti) { ti.set(termInfo); } - + + /** + * Returns document ids a term is located in + */ + public final int[] docs() { + return termInfo.docs; + } + /** Returns the docFreq from the current TermInfo in the enumeration. Initially invalid, valid after next() called for the first time.*/ public final int docFreq() { Index: org/apache/lucene/index/TermEnum.java =================================================================== --- org/apache/lucene/index/TermEnum.java (revision 653443) +++ org/apache/lucene/index/TermEnum.java (working copy) @@ -37,6 +37,8 @@ /** Closes the enumeration to further activity, freeing resources. */ public abstract void close() throws IOException; + public abstract int[] docs(); + // Term Vector support /** Skips terms to the first beyond the current whose value is Index: org/apache/lucene/index/TermInfo.java =================================================================== --- org/apache/lucene/index/TermInfo.java (revision 653443) +++ org/apache/lucene/index/TermInfo.java (working copy) @@ -26,6 +26,7 @@ long freqPointer = 0; long proxPointer = 0; int skipOffset; + int[] docs; TermInfo() {} @@ -40,14 +41,16 @@ freqPointer = ti.freqPointer; proxPointer = ti.proxPointer; skipOffset = ti.skipOffset; + this.docs = ti.docs; } final void set(int docFreq, - long freqPointer, long proxPointer, int skipOffset) { + long freqPointer, long proxPointer, int skipOffset, int[] docs) { this.docFreq = docFreq; this.freqPointer = freqPointer; this.proxPointer = proxPointer; this.skipOffset = skipOffset; + this.docs = docs; } final void set(TermInfo ti) { @@ -55,5 +58,6 @@ freqPointer = ti.freqPointer; proxPointer = ti.proxPointer; skipOffset = ti.skipOffset; + docs = ti.docs; } } Index: org/apache/lucene/index/TermInfosReader.java =================================================================== --- org/apache/lucene/index/TermInfosReader.java (revision 653443) +++ org/apache/lucene/index/TermInfosReader.java (working copy) @@ -59,12 +59,12 @@ fieldInfos = fis; origEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_EXTENSION, - readBufferSize), fieldInfos, false); + readBufferSize), fieldInfos, false, false); size = origEnum.size; totalIndexInterval = origEnum.indexInterval; indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, - readBufferSize), fieldInfos, true); + readBufferSize), fieldInfos, true, true); success = true; } finally { @@ -194,29 +194,35 @@ (indexOffset * totalIndexInterval) - 1, indexTerms[indexOffset], indexInfos[indexOffset]); } - + + TermInfo get(Term term) throws IOException { + return get(term, false); + } + /** Returns the TermInfo for a Term in the set, or null. */ - TermInfo get(Term term) throws IOException { - if (size == 0) return null; + TermInfo get(Term term, boolean loadDocs) throws IOException { + if (size == 0) + return null; - ensureIndexIsRead(); + ensureIndexIsRead(); - // optimize sequential access: first try scanning cached enum w/o seeking - SegmentTermEnum enumerator = getEnum(); - if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && term.compareTo(enumerator.prev())> 0) - || term.compareTo(enumerator.term()) >= 0)) { - int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; - if (indexTerms.length == enumOffset // but before end of block - || term.compareTo(indexTerms[enumOffset]) < 0) - return scanEnum(term); // no need to seek - } + // optimize sequential access: first try scanning cached enum w/o seeking + SegmentTermEnum enumerator = getEnum(); + enumerator.setLoadDocs(loadDocs); + if (enumerator.term() != null // term is at or past current + && ((enumerator.prev() != null && term.compareTo(enumerator.prev()) > 0) || term + .compareTo(enumerator.term()) >= 0)) { + int enumOffset = (int) (enumerator.position / totalIndexInterval) + 1; + if (indexTerms.length == enumOffset // but before end of block + || term.compareTo(indexTerms[enumOffset]) < 0) + return scanEnum(term); // no need to seek + } - // random-access: must seek - seekEnum(getIndexOffset(term)); - return scanEnum(term); - } - + // random-access: must seek + seekEnum(getIndexOffset(term)); + return scanEnum(term); + } + /** Scans within block for matching term. */ private final TermInfo scanEnum(Term term) throws IOException { SegmentTermEnum enumerator = getEnum(); @@ -271,10 +277,25 @@ public SegmentTermEnum terms() { return (SegmentTermEnum)origEnum.clone(); } - + + /** Returns an enumeration of all the Terms and TermInfos in the set. */ + public SegmentTermEnum terms(boolean loadDocs) { + SegmentTermEnum segmentTermEnum = (SegmentTermEnum) origEnum.clone(); + segmentTermEnum.setLoadDocs(true); + return segmentTermEnum; + } + /** Returns an enumeration of terms starting at or after the named term. */ public SegmentTermEnum terms(Term term) throws IOException { get(term); return (SegmentTermEnum)getEnum().clone(); } + + /** Returns an enumeration of terms starting at or after the named term. */ + public SegmentTermEnum terms(Term term, boolean loadDocs) throws IOException { + get(term, loadDocs); + SegmentTermEnum segmentTermEnum = (SegmentTermEnum) getEnum().clone(); + segmentTermEnum.setLoadDocs(loadDocs); + return segmentTermEnum; + } } Index: org/apache/lucene/index/TermInfosWriter.java =================================================================== --- org/apache/lucene/index/TermInfosWriter.java (revision 653443) +++ org/apache/lucene/index/TermInfosWriter.java (working copy) @@ -19,9 +19,12 @@ import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Directory; import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.VIntUtil; /** This stores a monotonically increasing set of pairs in a Directory. A TermInfos can be written once, in order. */ @@ -169,7 +172,15 @@ other.add(lastFieldNumber, lastTermBytes, lastTermBytesLength, lastTi); // add an index term writeTerm(fieldNumber, termBytes, termBytesLength); // write term - + if (ti.docs == null) { + output.writeVInt(-1); + } else { + Arrays.sort(ti.docs); + byte[] bytes = VIntUtil.toBytes(ti.docs); + output.writeVInt(bytes.length); + output.writeVInt(ti.docs.length); + output.writeBytes(bytes, bytes.length); + } output.writeVInt(ti.docFreq); // write doc freq output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers output.writeVLong(ti.proxPointer - lastTi.proxPointer); Index: org/apache/lucene/search/FilteredTermEnum.java =================================================================== --- org/apache/lucene/search/FilteredTermEnum.java (revision 653443) +++ org/apache/lucene/search/FilteredTermEnum.java (working copy) @@ -40,6 +40,10 @@ /** Indicates the end of the enumeration has been reached */ protected abstract boolean endEnum(); + public int[] docs() { + return actualEnum.docs(); + } + protected void setEnum(TermEnum actualEnum) throws IOException { this.actualEnum = actualEnum; // Find the first term that matches Index: org/apache/lucene/util/SortedVIntList.java =================================================================== --- org/apache/lucene/util/SortedVIntList.java (revision 653443) +++ org/apache/lucene/util/SortedVIntList.java (working copy) @@ -39,7 +39,17 @@ private int size; private byte[] bytes; private int lastBytePos; - + + SortedVIntList(int size, byte[] bytes) { + this.size = size; + this.bytes = bytes; + lastBytePos = bytes.length; + } + + byte[] getBytes() { + return bytes; + } + /** * Create a SortedVIntList from all elements of an array of integers. * @@ -105,7 +115,6 @@ builder.done(); } - private class SortedVIntListBuilder { private int lastInt = 0; @@ -166,7 +175,7 @@ public int size() { return size; } - + /** * @return The size of the byte array storing the compressed sorted integers. */