TermVector.YESFilterIndexReader contains another IndexReader, which it
* uses as its basic source of data, possibly transforming the data along the
* way or providing additional functionality. The class
@@ -87,6 +88,7 @@
public boolean next() throws IOException { return in.next(); }
public Term term() { return in.term(); }
public int docFreq() { return in.docFreq(); }
+ public DocIdSetIterator docs() throws IOException { return in.docs(); }
public void close() throws IOException { in.close(); }
}
@@ -177,7 +179,17 @@
protected void doSetNorm(int d, String f, byte b) throws CorruptIndexException, IOException {
in.setNorm(d, f, b);
}
+
+ public TermEnum terms(boolean loadDocs) throws IOException {
+ ensureOpen();
+ return in.terms(loadDocs);
+ }
+ public TermEnum terms(Term t, boolean loadDocs) throws IOException {
+ ensureOpen();
+ return in.terms(t, loadDocs);
+ }
+
public TermEnum terms() throws IOException {
ensureOpen();
return in.terms();
Index: org/apache/lucene/index/IndexReader.java
===================================================================
--- org/apache/lucene/index/IndexReader.java (revision 653899)
+++ org/apache/lucene/index/IndexReader.java (working copy)
@@ -89,6 +89,8 @@
public static final FieldOption TERMVECTOR_WITH_OFFSET = new FieldOption ("TERMVECTOR_WITH_OFFSET");
/** All fields with termvectors with offset values and position values enabled */
public static final FieldOption TERMVECTOR_WITH_POSITION_OFFSET = new FieldOption ("TERMVECTOR_WITH_POSITION_OFFSET");
+
+ public static final FieldOption TERM_DOCS = new FieldOption ("TERMVECTOR_DOCS");
}
private boolean closed;
@@ -645,6 +647,10 @@
* @throws IOException if there is a low-level IO error
*/
public abstract TermEnum terms(Term t) throws IOException;
+
+ public abstract TermEnum terms(Term t, boolean loadDocs) throws IOException;
+
+ public abstract TermEnum terms(boolean loadDocs) throws IOException;
/** Returns the number of documents containing the term t.
* @throws IOException if there is a low-level IO error
Index: org/apache/lucene/index/IndexWriter.java
===================================================================
--- org/apache/lucene/index/IndexWriter.java (revision 653899)
+++ org/apache/lucene/index/IndexWriter.java (working copy)
@@ -306,9 +306,6 @@
private SegmentInfos rollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails
private HashMap rollbackSegments;
- volatile SegmentInfos pendingCommit; // set when a commit is pending (after prepareCommit() & before commit())
- volatile long pendingCommitChangeCount;
-
private SegmentInfos localRollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails
private boolean localAutoCommit; // saved autoCommit during local transaction
private int localFlushedDocCount; // saved docWriter.getFlushedDocCount during local transaction
@@ -367,13 +364,12 @@
infoStream.println("IW " + messageID + " [" + Thread.currentThread().getName() + "]: " + message);
}
- private synchronized void setMessageID(PrintStream infoStream) {
+ private synchronized void setMessageID() {
if (infoStream != null && messageID == -1) {
synchronized(MESSAGE_ID_LOCK) {
messageID = MESSAGE_ID++;
}
}
- this.infoStream = infoStream;
}
/**
@@ -1086,8 +1082,9 @@
this.closeDir = closeDir;
directory = d;
analyzer = a;
- setMessageID(defaultInfoStream);
+ this.infoStream = defaultInfoStream;
this.maxFieldLength = maxFieldLength;
+ setMessageID();
if (create) {
// Clear the write lock in case it's leftover:
@@ -1499,7 +1496,8 @@
*/
public void setInfoStream(PrintStream infoStream) {
ensureOpen();
- setMessageID(infoStream);
+ this.infoStream = infoStream;
+ setMessageID();
docWriter.setInfoStream(infoStream);
deleter.setInfoStream(infoStream);
if (infoStream != null)
@@ -1674,7 +1672,7 @@
if (infoStream != null)
message("now call final commit()");
- commit(0);
+ commit(true, 0);
if (infoStream != null)
message("at close: " + segString());
@@ -2573,7 +2571,7 @@
if (autoCommit) {
boolean success = false;
try {
- commit(0);
+ commit(true, 0);
success = true;
} finally {
if (!success) {
@@ -2590,40 +2588,24 @@
}
/**
- * @deprecated Please use {@link #rollback} instead.
- */
- public void abort() throws IOException {
- rollback();
- }
-
- /**
* Close the IndexWriter without committing
* any of the changes that have occurred since it was
* opened. This removes any temporary files that had been
* created, after which the state of the index will be the
* same as it was when this writer was first opened. This
* can only be called when this IndexWriter was opened
- * with autoCommit=false. This also clears a
- * previous call to {@link #prepareCommit}.
+ * with autoCommit=false.
* @throws IllegalStateException if this is called when
* the writer was opened with autoCommit=true.
* @throws IOException if there is a low-level IO error
*/
- public void rollback() throws IOException {
+ public void abort() throws IOException {
ensureOpen();
if (autoCommit)
throw new IllegalStateException("abort() can only be called when IndexWriter was opened with autoCommit=false");
boolean doClose;
synchronized(this) {
-
- if (pendingCommit != null) {
- pendingCommit.rollbackCommit(directory);
- deleter.decRef(pendingCommit);
- pendingCommit = null;
- notifyAll();
- }
-
// Ensure that only one thread actually gets to do the closing:
if (!closing) {
doClose = true;
@@ -3131,54 +3113,10 @@
flush(true, false, true);
}
- /** Expert: prepare for commit. This does the first - * phase of 2-phase commit. You can only call this when - * autoCommit is false. This method does all steps - * necessary to commit changes since this writer was - * opened: flushes pending added and deleted docs, syncs - * the index files, writes most of next segments_N file. - * After calling this you must call either {@link - * #commit()} to finish the commit, or {@link - * #rollback()} to revert the commit and undo all changes - * done since the writer was opened.
- * - * You can also just call {@link #commit()} directly - * without prepareCommit first in which case that method - * will internally call prepareCommit. - */ - public final void prepareCommit() throws CorruptIndexException, IOException { - prepareCommit(false); - } - - private final void prepareCommit(boolean internal) throws CorruptIndexException, IOException { - - if (hitOOM) - throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot commit"); - - if (autoCommit && !internal) - throw new IllegalStateException("this method can only be used when autoCommit is false"); - - if (!autoCommit && pendingCommit != null) - throw new IllegalStateException("prepareCommit was already called with no corresponding call to commit"); - - message("prepareCommit: flush"); - - flush(true, true, true); - - startCommit(0); - } - - private void commit(long sizeInBytes) throws IOException { - startCommit(sizeInBytes); - finishCommit(); - } - /** - *Commits all pending updates (added & deleted - * documents) to the index, and syncs all referenced index - * files, such that a reader will see the changes and the - * index updates will survive an OS or machine crash or - * power loss (though, see the note below). Note that + *
Commits all pending updates (added & deleted documents) + * to the index, and syncs all referenced index files, + * such that a reader will see the changes. Note that * this does not wait for any running background merges to * finish. This may be a costly operation, so you should * test the cost in your application and do it only when @@ -3197,38 +3135,12 @@ * consistency on such devices.
*/ public final void commit() throws CorruptIndexException, IOException { - - message("commit: start"); - - if (autoCommit || pendingCommit == null) { - message("commit: now prepare"); - prepareCommit(true); - } else - message("commit: already prepared"); - - finishCommit(); + commit(true); } - private synchronized final void finishCommit() throws CorruptIndexException, IOException { - - if (pendingCommit != null) { - try { - message("commit: pendingCommit != null"); - pendingCommit.finishCommit(directory); - lastCommitChangeCount = pendingCommitChangeCount; - segmentInfos.updateGeneration(pendingCommit); - setRollbackSegmentInfos(); - deleter.checkpoint(pendingCommit, true); - } finally { - deleter.decRef(pendingCommit); - pendingCommit = null; - notifyAll(); - } - - } else - message("commit: pendingCommit == null; skip"); - - message("commit: done"); + private final void commit(boolean triggerMerges) throws CorruptIndexException, IOException { + flush(triggerMerges, true, true); + commit(true, 0); } /** @@ -3264,7 +3176,8 @@ // when flushing a segment; otherwise deletes may become // visible before their corresponding added document // from an updateDocument call - flushDeletes |= autoCommit; + if (autoCommit) + flushDeletes = true; // Returns true if docWriter is currently aborting, in // which case we skip flushing this segment @@ -4022,7 +3935,7 @@ synchronized(this) { size = merge.info.sizeInBytes(); } - commit(size); + commit(false, size); } success = false; @@ -4075,7 +3988,7 @@ synchronized(this) { size = merge.info.sizeInBytes(); } - commit(size); + commit(false, size); } return mergedDocCount; @@ -4238,13 +4151,13 @@ } /** Walk through all files referenced by the current - * segmentInfos and ask the Directory to sync each file, - * if it wasn't already. If that succeeds, then we - * prepare a new segments_N file but do not fully commit - * it. */ - private void startCommit(long sizeInBytes) throws IOException { + * segmentInfos, minus flushes, and ask the Directory to + * sync each file, if it wasn't already. If that + * succeeds, then we write a new segments_N file & sync + * that. */ + private void commit(boolean skipWait, long sizeInBytes) throws IOException { - assert testPoint("startStartCommit"); + assert testPoint("startCommit"); if (hitOOM) return; @@ -4252,9 +4165,9 @@ try { if (infoStream != null) - message("startCommit(): start sizeInBytes=" + sizeInBytes); + message("start commit() skipWait=" + skipWait + " sizeInBytes=" + sizeInBytes); - if (sizeInBytes > 0) + if (!skipWait) syncPause(sizeInBytes); SegmentInfos toSync = null; @@ -4266,7 +4179,7 @@ if (changeCount == lastCommitChangeCount) { if (infoStream != null) - message(" skip startCommit(): no changes pending"); + message(" skip commit(): no changes pending"); return; } @@ -4276,17 +4189,15 @@ // threads can be doing this at once, if say a large // merge and a small merge finish at the same time: - if (infoStream != null) - message("startCommit index=" + segString(segmentInfos) + " changeCount=" + changeCount); - toSync = (SegmentInfos) segmentInfos.clone(); deleter.incRef(toSync, false); myChangeCount = changeCount; } - assert testPoint("midStartCommit"); + if (infoStream != null) + message("commit index=" + segString(toSync)); - boolean setPending = false; + assert testPoint("midCommit"); try { @@ -4326,72 +4237,54 @@ break; } - assert testPoint("midStartCommit2"); - + assert testPoint("midCommit2"); + synchronized(this) { // If someone saved a newer version of segments file // since I first started syncing my version, I can // safely skip saving myself since I've been // superseded: - if (myChangeCount > lastCommitChangeCount && (pendingCommit == null || myChangeCount > pendingCommitChangeCount)) { - - // Wait now for any current pending commit to complete: - while(pendingCommit != null) { - message("wait for existing pendingCommit to finish..."); - try { - wait(); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - } - + if (myChangeCount > lastCommitChangeCount) { + if (segmentInfos.getGeneration() > toSync.getGeneration()) toSync.updateGeneration(segmentInfos); boolean success = false; try { - - // Exception here means nothing is prepared - // (this method unwinds everything it did on - // an exception) - try { - toSync.prepareCommit(directory); - } finally { - // Have our master segmentInfos record the - // generations we just prepared. We do this - // on error or success so we don't - // double-write a segments_N file. - segmentInfos.updateGeneration(toSync); - } - - assert pendingCommit == null; - setPending = true; - pendingCommit = toSync; - pendingCommitChangeCount = myChangeCount; + toSync.commit(directory); success = true; } finally { + // Have our master segmentInfos record the + // generations we just sync'd + segmentInfos.updateGeneration(toSync); if (!success) message("hit exception committing segments file"); } + + message("commit complete"); + + lastCommitChangeCount = myChangeCount; + + deleter.checkpoint(toSync, true); + setRollbackSegmentInfos(); } else message("sync superseded by newer infos"); } message("done all syncs"); - assert testPoint("midStartCommitSuccess"); + assert testPoint("midCommitSuccess"); } finally { synchronized(this) { - if (!setPending) - deleter.decRef(toSync); + deleter.decRef(toSync); } } } catch (OutOfMemoryError oom) { hitOOM = true; throw oom; } - assert testPoint("finishStartCommit"); + assert testPoint("finishCommit"); } /** @@ -4484,11 +4377,11 @@ // Used only by assert for testing. Current points: // startDoFlush // startCommitMerge - // startStartCommit - // midStartCommit - // midStartCommit2 - // midStartCommitSuccess - // finishStartCommit + // startCommit + // midCommit + // midCommit2 + // midCommitSuccess + // finishCommit // startCommitMergeDeletes // startMergeInit // startApplyDeletes Index: org/apache/lucene/index/MultiReader.java =================================================================== --- org/apache/lucene/index/MultiReader.java (revision 653899) +++ org/apache/lucene/index/MultiReader.java (working copy) @@ -295,14 +295,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, null); + return new MultiTermEnum(subReaders, starts, null, false); } public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, term); + return new MultiTermEnum(subReaders, starts, term, false); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, null, loadDocs); + } + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, term, loadDocs); + } + public int docFreq(Term t) throws IOException { ensureOpen(); int total = 0; // sum freqs in segments Index: org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- org/apache/lucene/index/MultiSegmentReader.java (revision 653899) +++ org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -17,20 +17,24 @@ * limitations under the License. */ -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.store.Directory; - import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Hashtable; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; -/** +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.Directory; + +/** * An IndexReader which reads indexes with multiple segments. */ class MultiSegmentReader extends DirectoryIndexReader { @@ -358,14 +362,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, null); + return new MultiTermEnum(subReaders, starts, null, false); } + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, null, loadDocs); + } + public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, term); + return new MultiTermEnum(subReaders, starts, term, false); } + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, term, loadDocs); + } + public int docFreq(Term t) throws IOException { ensureOpen(); int total = 0; // sum freqs in segments @@ -449,18 +463,22 @@ private Term term; private int docFreq; + private List matchingDocs = new ArrayList(10); + private boolean loadDocs; + private MultiDocsIterator multiDocsIterator; - public MultiTermEnum(IndexReader[] readers, int[] starts, Term t) + public MultiTermEnum(IndexReader[] readers, int[] starts, Term t, boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; queue = new SegmentMergeQueue(readers.length); for (int i = 0; i < readers.length; i++) { IndexReader reader = readers[i]; TermEnum termEnum; if (t != null) { - termEnum = reader.terms(t); + termEnum = reader.terms(t, loadDocs); } else - termEnum = reader.terms(); + termEnum = reader.terms(loadDocs); SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader); if (t == null ? smi.next() : termEnum.term() != null) @@ -473,7 +491,16 @@ next(); } } - + + public final DocIdSetIterator docs() throws IOException { + if (multiDocsIterator == null) { + multiDocsIterator = new MultiDocsIterator(matchingDocs, this); + } else { + multiDocsIterator.set(matchingDocs); + } + return multiDocsIterator; + } + public boolean next() throws IOException { SegmentMergeInfo top = (SegmentMergeInfo)queue.top(); if (top == null) { @@ -483,10 +510,16 @@ term = top.term; docFreq = 0; - + matchingDocs.clear(); while (top != null && term.compareTo(top.term) == 0) { queue.pop(); docFreq += top.termEnum.docFreq(); // increment freq + + DocIdSetIterator docsIterator = top.termEnum.docs(); + if (docsIterator != null) { + matchingDocs.add(new DocIdSetHolder(top.base, docsIterator)); + } + if (top.next()) queue.put(top); // restore queue else @@ -508,7 +541,69 @@ queue.close(); } } + + private static class DocIdSetHolder implements Comparable { + private int base; + private DocIdSetIterator docsIterator; + + public DocIdSetHolder(int base, DocIdSetIterator docsIterator) { + this.base = base; + this.docsIterator = docsIterator; + } + + public int compareTo(Object object) { + DocIdSetHolder other = (DocIdSetHolder)object; + if (base < other.base) return 1; + if (base > other.base) return -1; + return 0; + } + } + + private static class MultiDocsIterator extends DocIdSetIterator { + private List iteratorList; + private int doc; + private MultiTermEnum multiTermEnum; + + public MultiDocsIterator(List iteratorList, MultiTermEnum multiTermEnum) throws IOException { + this.iteratorList = new ArrayList(iteratorList); + this.multiTermEnum = multiTermEnum; + // keeps docs in ascending order by sorting by segment base + Collections.sort(iteratorList); + } + + private void set(List iteratorList) { + this.iteratorList.clear(); + for (int x=0; x < iteratorList.size(); x++) { + this.iteratorList.add(iteratorList.get(x)); + } + // keeps docs in ascending order by sorting by segment base + Collections.sort(iteratorList); + } + + public int doc() { + return doc; + } + + public boolean next() throws IOException { + if (iteratorList.size() == 0) { + doc = -1; + return false; + } + DocIdSetHolder holder = (DocIdSetHolder)iteratorList.get(0); + DocIdSetIterator docsIterator = holder.docsIterator; + if (!docsIterator.next()) { + iteratorList.remove(0); + return next(); + } + doc = docsIterator.doc() + holder.base; + return true; + } + public boolean skipTo(int target) throws IOException { + throw new UnsupportedOperationException(""); + } + } + static class MultiTermDocs implements TermDocs { protected IndexReader[] readers; protected int[] starts; Index: org/apache/lucene/index/ParallelReader.java =================================================================== --- org/apache/lucene/index/ParallelReader.java (revision 653899) +++ org/apache/lucene/index/ParallelReader.java (working copy) @@ -17,15 +17,25 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.DocIdSetIterator; -import java.io.IOException; -import java.util.*; - /** An IndexReader which reads multiple, parallel indexes. Each index added * must have the same number of documents, but typically each contains * different fields. Each document contains the union of the fields of all @@ -350,14 +360,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new ParallelTermEnum(); + return new ParallelTermEnum(false); } public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new ParallelTermEnum(term); + return new ParallelTermEnum(term, false); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new ParallelTermEnum(loadDocs); + } + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new ParallelTermEnum(term, loadDocs); + } + public int docFreq(Term term) throws IOException { ensureOpen(); IndexReader reader = ((IndexReader)fieldToReader.get(term.field())); @@ -455,18 +475,21 @@ private String field; private Iterator fieldIterator; private TermEnum termEnum; + private boolean loadDocs; - public ParallelTermEnum() throws IOException { + public ParallelTermEnum(boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; field = (String)fieldToReader.firstKey(); if (field != null) - termEnum = ((IndexReader)fieldToReader.get(field)).terms(); + termEnum = ((IndexReader)fieldToReader.get(field)).terms(loadDocs); } - public ParallelTermEnum(Term term) throws IOException { + public ParallelTermEnum(Term term, boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; field = term.field(); IndexReader reader = ((IndexReader)fieldToReader.get(field)); if (reader!=null) - termEnum = reader.terms(term); + termEnum = reader.terms(term, loadDocs); } public boolean next() throws IOException { @@ -510,7 +533,12 @@ return termEnum.docFreq(); } - + + public DocIdSetIterator docs() throws IOException { + if (termEnum == null) return null; + return termEnum.docs(); + } + public void close() throws IOException { if (termEnum!=null) termEnum.close(); Index: org/apache/lucene/index/SegmentMerger.java =================================================================== --- org/apache/lucene/index/SegmentMerger.java (revision 653899) +++ org/apache/lucene/index/SegmentMerger.java (working copy) @@ -17,16 +17,19 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; import java.util.Vector; -import java.util.Iterator; -import java.util.Collection; -import java.io.IOException; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IntArrayList; +import org.apache.lucene.util.SortedVIntList; /** * The SegmentMerger class combines two or more Segments, represented by an IndexReader ({@link #add}, @@ -64,7 +67,9 @@ /** Maximum number of contiguous documents to bulk-copy when merging stored fields */ private final static int MAX_RAW_MERGE_DOCS = 4192; - + + private IntArrayList termDocsList = new IntArrayList(50, 1.25); + /** This ctor used only by test code. * * @param dir The Directory to merge the other segments into @@ -197,11 +202,11 @@ } private void addIndexed(IndexReader reader, FieldInfos fieldInfos, Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector, boolean storePayloads) throws IOException { + boolean storeOffsetWithTermVector, boolean storePayloads, boolean storeTermDocs) throws IOException { Iterator i = names.iterator(); while (i.hasNext()) { String field = (String)i.next(); - fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads); + fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads, storeTermDocs); } } @@ -264,15 +269,17 @@ SegmentReader segmentReader = (SegmentReader) reader; for (int j = 0; j < segmentReader.getFieldInfos().size(); j++) { FieldInfo fi = segmentReader.getFieldInfos().fieldInfo(j); - fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads); + fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads, fi.storeTermDoc); } } else { - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERM_DOCS), false, false, false, false, true); + fieldInfos.add(reader.getFieldNames(IndexReader.FieldOption.UNINDEXED), false); } } @@ -502,7 +509,7 @@ top = (SegmentMergeInfo) queue.top(); } - final int df = mergeTermInfo(match, matchSize); // add new TermInfo + final int df = mergeTermInfo(match, matchSize, term); // add new TermInfo if (checkAbort != null) checkAbort.work(df/3.0); @@ -528,24 +535,43 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int mergeTermInfo(SegmentMergeInfo[] smis, int n) + private final int mergeTermInfo(SegmentMergeInfo[] smis, int n, Term term) throws CorruptIndexException, IOException { long freqPointer = freqOutput.getFilePointer(); long proxPointer = proxOutput.getFilePointer(); + + Holder docsHolder = new Holder(null); + // TODO: can optimize to not generate doc array if no need + int df = appendPostings(smis, n, docsHolder); // append posting data - int df = appendPostings(smis, n); // append posting data - long skipPointer = skipListWriter.writeSkip(freqOutput); if (df > 0) { + int[] docs = null; + FieldInfo fieldInfo = fieldInfos.fieldInfo(term.field()); + SortedVIntList vintList = null; + if (fieldInfo.storeTermDoc) { + docs = (int[])docsHolder.object; + Arrays.sort(docs); + vintList = new SortedVIntList(docs); + } + // add an entry to the dictionary with pointers to prox and freq files - termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer)); + termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer), vintList); termInfosWriter.add(smis[0].term, termInfo); } return df; } + public static class Holder { + public Object object; + + public Holder(Object object) { + this.object = object; + } + } + private byte[] payloadBuffer; private int[][] docMaps; int[][] getDocMaps() { @@ -566,13 +592,14 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int appendPostings(SegmentMergeInfo[] smis, int n) + private final int appendPostings(SegmentMergeInfo[] smis, int n, Holder docsHolder) throws CorruptIndexException, IOException { int lastDoc = 0; int df = 0; // number of docs w/ term skipListWriter.resetSkip(); boolean storePayloads = fieldInfos.fieldInfo(smis[0].term.field).storePayloads; int lastPayloadLength = -1; // ensures that we write the first length + termDocsList.clear(); for (int i = 0; i < n; i++) { SegmentMergeInfo smi = smis[i]; TermPositions postings = smi.getPositions(); @@ -589,7 +616,9 @@ if (doc < 0 || (df > 0 && doc <= lastDoc)) throw new CorruptIndexException("docs out of order (" + doc + " <= " + lastDoc + " )"); - + + termDocsList.add(doc); + df++; if ((df % skipInterval) == 0) { @@ -638,6 +667,7 @@ } } } + docsHolder.object = termDocsList.toIntArray(); return df; } Index: org/apache/lucene/index/SegmentReader.java =================================================================== --- org/apache/lucene/index/SegmentReader.java (revision 653899) +++ org/apache/lucene/index/SegmentReader.java (working copy) @@ -653,7 +653,17 @@ ensureOpen(); return tis.terms(t); } + + public TermEnum terms(boolean loadDocs) { + ensureOpen(); + return tis.terms(loadDocs); + } + public TermEnum terms(Term t, boolean loadDocs) throws IOException { + ensureOpen(); + return tis.terms(t, loadDocs); + } + FieldInfos getFieldInfos() { return fieldInfos; } @@ -756,6 +766,8 @@ else if ((fi.storeOffsetWithTermVector && fi.storePositionWithTermVector) && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) { fieldSet.add(fi.name); + } else if (fi.storeTermDoc) { + fieldSet.add(fi.name); } } return fieldSet; Index: org/apache/lucene/index/SegmentTermEnum.java =================================================================== --- org/apache/lucene/index/SegmentTermEnum.java (revision 653899) +++ org/apache/lucene/index/SegmentTermEnum.java (working copy) @@ -18,9 +18,14 @@ */ import java.io.IOException; + +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.SortedVIntList; +import org.apache.lucene.util.VIntUtil; final class SegmentTermEnum extends TermEnum implements Cloneable { + public static final int TERM_DOCS_BYTES_THRESHOLD = 50; private IndexInput input; FieldInfos fieldInfos; long size; @@ -29,6 +34,7 @@ private TermBuffer termBuffer = new TermBuffer(); private TermBuffer prevBuffer = new TermBuffer(); private TermBuffer scanBuffer = new TermBuffer(); // used for scanning + private byte[] termDocsBuffer = new byte[TERM_DOCS_BYTES_THRESHOLD]; private TermInfo termInfo = new TermInfo(); @@ -39,14 +45,18 @@ int skipInterval; int maxSkipLevels; private int formatM1SkipInterval; + private boolean loadDocs = false; + long nextFilePointer = -1; + IndexInput docsInput; - SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi) - throws CorruptIndexException, IOException { + SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi, boolean loadDocs) + throws CorruptIndexException, IOException { input = i; fieldInfos = fis; isIndex = isi; - maxSkipLevels = 1; // use single-level skip lists for formats > -3 - + this.loadDocs = loadDocs; + maxSkipLevels = 1; // use single-level skip lists for formats > -3 + int firstInt = input.readInt(); if (firstInt >= 0) { // original-format file, without explicit format version number @@ -90,19 +100,23 @@ } } + void setLoadDocs(boolean loadDocs) { + this.loadDocs = loadDocs; + } + protected Object clone() { SegmentTermEnum clone = null; try { clone = (SegmentTermEnum) super.clone(); - } catch (CloneNotSupportedException e) {} - + } catch (CloneNotSupportedException e) { + } + clone.loadDocs = loadDocs; clone.input = (IndexInput) input.clone(); clone.termInfo = new TermInfo(termInfo); clone.termBuffer = (TermBuffer)termBuffer.clone(); clone.prevBuffer = (TermBuffer)prevBuffer.clone(); clone.scanBuffer = new TermBuffer(); - return clone; } @@ -122,7 +136,7 @@ termBuffer.reset(); return false; } - + prevBuffer.set(termBuffer); termBuffer.read(input, fieldInfos); @@ -145,11 +159,80 @@ } if (isIndex) - indexPointer += input.readVLong(); // read index pointer + indexPointer += input.readVLong(); // read index pointer + // read docs + if (!isIndex) { + termInfo.numDocBytes = input.readVInt(); + termInfo.docsPointer = input.getFilePointer(); + if (termInfo.numDocBytes > 0) { + // lower than threshold, cache in terminfo + if (termInfo.numDocBytes <= TERM_DOCS_BYTES_THRESHOLD) { + byte[] bytes = new byte[termInfo.numDocBytes]; + input.readBytes(bytes, 0, termInfo.numDocBytes); + termInfo.docs = new SortedVIntList(termInfo.docFreq, bytes); + } else { + // will use iterator over file + input.seek(termInfo.numDocBytes + termInfo.docsPointer); + } + } + } return true; } + private static class DocIdSetReader extends DocIdSetIterator { + private static final int VB1 = 0x7F; + private static final int BIT_SHIFT = 7; + private final int MAX_BYTES_PER_INT = (31 / BIT_SHIFT) + 1; + private int bytePos = 0; + private int lastBytePos; + int lastInt = 0; + private boolean isValid = true; + private IndexInput indexInput; + + public DocIdSetReader(IndexInput indexInput, int lastBytePos) { + this.indexInput = indexInput; + this.lastBytePos = lastBytePos; + } + + private void advance() throws IOException { + try { + byte b = indexInput.readByte(); + bytePos++; + lastInt += b & VB1; + for (int s = BIT_SHIFT; (b & ~VB1) != 0; s += BIT_SHIFT) { + b = indexInput.readByte(); + bytePos++; + lastInt += (b & VB1) << s; + } + } catch (IOException ioException) { + throw new IOException("file pointer: "+indexInput.getFilePointer()+" bytepos: "+bytePos+" lastBytePos: "+lastBytePos, ioException); + } + } + + public int doc() { + if (!isValid) { + throw new RuntimeException("invalid"); + } + return lastInt; } + + public boolean next() throws IOException { + if (!isValid) { + throw new IOException("invalid"); + } + if (bytePos >= lastBytePos) { + return false; + } else { + advance(); + return true; + } + } + + public boolean skipTo(int target) throws IOException { + throw new UnsupportedOperationException(""); + } + } + /** Optimized scan, without allocating new terms. */ final void scanTo(Term term) throws IOException { scanBuffer.set(term); @@ -179,20 +262,47 @@ ti.set(termInfo); } - /** Returns the docFreq from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ + /** + * Returns document ids a term is located in + */ + public final DocIdSetIterator docs() throws IOException { + if (termInfo.numDocBytes > 0) { + if (termInfo.numDocBytes < TERM_DOCS_BYTES_THRESHOLD) { + return termInfo.docs.iterator(); + } else { + // read from file + if (docsInput == null) { + docsInput = (IndexInput)input.clone(); + } + docsInput.seek(termInfo.docsPointer); + DocIdSetReader docIdSetReader = new DocIdSetReader(docsInput, termInfo.numDocBytes); + return docIdSetReader; + } + } else { + return null; + } + } + + /** + * Returns the docFreq from the current TermInfo in the enumeration. Initially + * invalid, valid after next() called for the first time. + */ public final int docFreq() { return termInfo.docFreq; } - /* Returns the freqPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ + /* + * Returns the freqPointer from the current TermInfo in the enumeration. + * Initially invalid, valid after next() called for the first time. + */ final long freqPointer() { return termInfo.freqPointer; } - /* Returns the proxPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ + /* + * Returns the proxPointer from the current TermInfo in the enumeration. + * Initially invalid, valid after next() called for the first time. + */ final long proxPointer() { return termInfo.proxPointer; } @@ -200,5 +310,6 @@ /** Closes the enumeration to further activity, freeing resources. */ public final void close() throws IOException { input.close(); + if (docsInput != null) docsInput.close(); } } Index: org/apache/lucene/index/TermEnum.java =================================================================== --- org/apache/lucene/index/TermEnum.java (revision 653899) +++ org/apache/lucene/index/TermEnum.java (working copy) @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.search.DocIdSetIterator; + /** Abstract class for enumerating terms.Term enumerations are always ordered by Term.compareTo(). Each term in
@@ -37,6 +39,9 @@
/** Closes the enumeration to further activity, freeing resources. */
public abstract void close() throws IOException;
+ /** Returns an iterator over the doc ids **/
+ public abstract DocIdSetIterator docs() throws IOException;
+
// Term Vector support
/** Skips terms to the first beyond the current whose value is
Index: org/apache/lucene/index/TermInfo.java
===================================================================
--- org/apache/lucene/index/TermInfo.java (revision 653899)
+++ org/apache/lucene/index/TermInfo.java (working copy)
@@ -1,5 +1,7 @@
package org.apache.lucene.index;
+import org.apache.lucene.util.SortedVIntList;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -26,6 +28,9 @@
long freqPointer = 0;
long proxPointer = 0;
int skipOffset;
+ long docsPointer = 0;
+ int numDocBytes;
+ SortedVIntList docs;
TermInfo() {}
@@ -40,14 +45,18 @@
freqPointer = ti.freqPointer;
proxPointer = ti.proxPointer;
skipOffset = ti.skipOffset;
+ this.docs = ti.docs;
+ this.docsPointer = ti.docsPointer;
+ this.numDocBytes = ti.numDocBytes;
}
final void set(int docFreq,
- long freqPointer, long proxPointer, int skipOffset) {
+ long freqPointer, long proxPointer, int skipOffset, SortedVIntList docs) {
this.docFreq = docFreq;
this.freqPointer = freqPointer;
this.proxPointer = proxPointer;
this.skipOffset = skipOffset;
+ this.docs = docs;
}
final void set(TermInfo ti) {
@@ -55,5 +64,8 @@
freqPointer = ti.freqPointer;
proxPointer = ti.proxPointer;
skipOffset = ti.skipOffset;
+ docs = ti.docs;
+ docsPointer = ti.docsPointer;
+ numDocBytes = ti.numDocBytes;
}
}
Index: org/apache/lucene/index/TermInfosReader.java
===================================================================
--- org/apache/lucene/index/TermInfosReader.java (revision 653899)
+++ org/apache/lucene/index/TermInfosReader.java (working copy)
@@ -59,12 +59,12 @@
fieldInfos = fis;
origEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_EXTENSION,
- readBufferSize), fieldInfos, false);
+ readBufferSize), fieldInfos, false, false);
size = origEnum.size;
totalIndexInterval = origEnum.indexInterval;
indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION,
- readBufferSize), fieldInfos, true);
+ readBufferSize), fieldInfos, true, true);
success = true;
} finally {
@@ -197,12 +197,17 @@
/** Returns the TermInfo for a Term in the set, or null. */
TermInfo get(Term term) throws IOException {
+ return get(term, false);
+ }
+
+ TermInfo get(Term term, boolean loadDocs) throws IOException {
if (size == 0) return null;
ensureIndexIsRead();
// optimize sequential access: first try scanning cached enum w/o seeking
SegmentTermEnum enumerator = getEnum();
+ enumerator.setLoadDocs(loadDocs);
if (enumerator.term() != null // term is at or past current
&& ((enumerator.prev() != null && term.compareTo(enumerator.prev())> 0)
|| term.compareTo(enumerator.term()) >= 0)) {
@@ -271,10 +276,25 @@
public SegmentTermEnum terms() {
return (SegmentTermEnum)origEnum.clone();
}
-
+
+ /** Returns an enumeration of all the Terms and TermInfos in the set. */
+ public SegmentTermEnum terms(boolean loadDocs) {
+ SegmentTermEnum segmentTermEnum = (SegmentTermEnum) origEnum.clone();
+ segmentTermEnum.setLoadDocs(true);
+ return segmentTermEnum;
+ }
+
/** Returns an enumeration of terms starting at or after the named term. */
public SegmentTermEnum terms(Term term) throws IOException {
get(term);
return (SegmentTermEnum)getEnum().clone();
}
+
+ /** Returns an enumeration of terms starting at or after the named term. */
+ public SegmentTermEnum terms(Term term, boolean loadDocs) throws IOException {
+ get(term, loadDocs);
+ SegmentTermEnum segmentTermEnum = (SegmentTermEnum) getEnum().clone();
+ segmentTermEnum.setLoadDocs(loadDocs);
+ return segmentTermEnum;
+ }
}
Index: org/apache/lucene/index/TermInfosWriter.java
===================================================================
--- org/apache/lucene/index/TermInfosWriter.java (revision 653899)
+++ org/apache/lucene/index/TermInfosWriter.java (working copy)
@@ -19,9 +19,12 @@
import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.VIntUtil;
/** This stores a monotonically increasing set of