Index: org/apache/lucene/document/AbstractField.java =================================================================== --- org/apache/lucene/document/AbstractField.java (revision 652841) +++ org/apache/lucene/document/AbstractField.java (working copy) @@ -32,6 +32,7 @@ protected boolean isTokenized = true; protected boolean isBinary = false; protected boolean isCompressed = false; + protected boolean isStoreTermDocs = false; protected boolean lazy = false; protected float boost = 1.0f; // the one and only data object for all different kind of field values @@ -42,7 +43,7 @@ } - protected AbstractField(String name, Field.Store store, Field.Index index, Field.TermVector termVector) { + protected AbstractField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, Field.TermDocs termDocs) { if (name == null) throw new NullPointerException("name cannot be null"); this.name = name.intern(); // field names are interned @@ -78,7 +79,15 @@ } else { throw new IllegalArgumentException("unknown index parameter " + index); } - + + if (termDocs == Field.TermDocs.STORE) { + isStoreTermDocs = true; + } else if (termDocs == Field.TermDocs.NO) { + isStoreTermDocs = false; + } else { + throw new IllegalArgumentException("unknown index parameter " + index); + } + this.isBinary = false; setStoreTermVector(termVector); @@ -159,7 +168,9 @@ with search hits. It is an error for this to be true if a field is Reader-valued. */ public final boolean isStored() { return isStored; } - + + public final boolean isStoreTermDocs() { return isStoreTermDocs; } + /** True iff the value of the field is to be indexed, so that it may be searched on. */ public final boolean isIndexed() { return isIndexed; } Index: org/apache/lucene/document/Field.java =================================================================== --- org/apache/lucene/document/Field.java (revision 652841) +++ org/apache/lucene/document/Field.java (working copy) @@ -56,14 +56,31 @@ /** Do not store the field value in the index. */ public static final Store NO = new Store("NO"); } - + /** Specifies whether and how a field should be indexed. */ + public static final class TermDocs extends Parameter implements Serializable { + private TermDocs(String name) { + super(name); + } + + /** + * Store docs with the term in the term dictionary + */ + public static final TermDocs STORE = new TermDocs("STORE"); + + /** + * Do not store docs with the term in the term dictionary + */ + public static final TermDocs NO = new TermDocs("NO"); + } + + /** Specifies whether and how a field should be indexed. */ public static final class Index extends Parameter implements Serializable { private Index(String name) { super(name); } - + /** Do not index the field value. This field can thus not be searched, * but one can still access its contents provided it is * {@link Field.Store stored}. */ @@ -202,9 +219,17 @@ * @throws IllegalArgumentException if the field is neither stored nor indexed */ public Field(String name, String value, Store store, Index index) { - this(name, value, store, index, TermVector.NO); + this(name, value, store, index, TermVector.NO, TermDocs.NO); } + public Field(String name, String value, Store store, Index index, TermDocs termDocs) { + this(name, value, store, index, TermVector.NO, termDocs); + } + + public Field(String name, String value, Store store, Index index, TermVector termVector) { + this(name, value, store, index, termVector, TermDocs.NO); + } + /** * Create a field by specifying its name, value and how it will * be saved in the index. @@ -222,7 +247,7 @@ *
  • the field is not indexed but termVector is TermVector.YES
    • * */ - public Field(String name, String value, Store store, Index index, TermVector termVector) { + public Field(String name, String value, Store store, Index index, TermVector termVector, TermDocs termDocs) { if (name == null) throw new NullPointerException("name cannot be null"); if (value == null) @@ -271,6 +296,14 @@ throw new IllegalArgumentException("unknown index parameter " + index); } + if (termDocs == TermDocs.NO) { + this.isStoreTermDocs = false; + } else if (termDocs == TermDocs.STORE) { + this.isStoreTermDocs = true; + } else { + throw new IllegalArgumentException("unknown termdocs parameter " + termDocs); + } + this.isBinary = false; setStoreTermVector(termVector); Index: org/apache/lucene/document/Fieldable.java =================================================================== --- org/apache/lucene/document/Fieldable.java (revision 652841) +++ org/apache/lucene/document/Fieldable.java (working copy) @@ -108,7 +108,9 @@ * @see org.apache.lucene.index.IndexReader#getTermFreqVector(int, String) */ boolean isTermVectorStored(); - + + boolean isStoreTermDocs(); + /** * True iff terms are stored as term vector together with their offsets * (start and end positon in source text). Index: org/apache/lucene/index/DocumentsWriter.java =================================================================== --- org/apache/lucene/index/DocumentsWriter.java (revision 652841) +++ org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -17,1618 +17,1695 @@ * limitations under the License. */ +import java.io.IOException; +import java.io.PrintStream; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Similarity; import org.apache.lucene.search.Weight; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.util.IntArrayList; import org.apache.lucene.util.UnicodeUtil; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.HashMap; -import java.util.ArrayList; -import java.util.Map.Entry; -import java.text.NumberFormat; -import java.util.Collections; - /** - * This class accepts multiple added documents and directly - * writes a single segment file. It does this more - * efficiently than creating a single segment per document - * (with DocumentWriter) and doing standard merges on those + * This class accepts multiple added documents and directly writes a single + * segment file. It does this more efficiently than creating a single segment + * per document (with DocumentWriter) and doing standard merges on those * segments. - * - * When a document is added, its stored fields (if any) and - * term vectors (if any) are immediately written to the - * Directory (ie these do not consume RAM). The freq/prox - * postings are accumulated into a Postings hash table keyed - * by term. Each entry in this hash table holds a separate - * byte stream (allocated as incrementally growing slices - * into large shared byte[] arrays) for freq and prox, that - * contains the postings data for multiple documents. If - * vectors are enabled, each unique term for each document - * also allocates a PostingVector instance to similarly - * track the offsets & positions byte stream. - * - * Once the Postings hash is full (ie is consuming the - * allowed RAM) or the number of added docs is large enough - * (in the case we are flushing by doc count instead of RAM - * usage), we create a real segment and flush it to disk and - * reset the Postings hash. - * - * In adding a document we first organize all of its fields - * by field name. We then process field by field, and - * record the Posting hash per-field. After each field we - * flush its term vectors. When it's time to flush the full - * segment we first sort the fields by name, and then go - * field by field and sorts its postings. - * - * + * + * When a document is added, its stored fields (if any) and term vectors (if + * any) are immediately written to the Directory (ie these do not consume RAM). + * The freq/prox postings are accumulated into a Postings hash table keyed by + * term. Each entry in this hash table holds a separate byte stream (allocated + * as incrementally growing slices into large shared byte[] arrays) for freq and + * prox, that contains the postings data for multiple documents. If vectors are + * enabled, each unique term for each document also allocates a PostingVector + * instance to similarly track the offsets & positions byte stream. + * + * Once the Postings hash is full (ie is consuming the allowed RAM) or the + * number of added docs is large enough (in the case we are flushing by doc + * count instead of RAM usage), we create a real segment and flush it to disk + * and reset the Postings hash. + * + * In adding a document we first organize all of its fields by field name. We + * then process field by field, and record the Posting hash per-field. After + * each field we flush its term vectors. When it's time to flush the full + * segment we first sort the fields by name, and then go field by field and + * sorts its postings. + * + * * Threads: - * - * Multiple threads are allowed into addDocument at once. - * There is an initial synchronized call to getThreadState - * which allocates a ThreadState for this thread. The same - * thread will get the same ThreadState over time (thread - * affinity) so that if there are consistent patterns (for - * example each thread is indexing a different content - * source) then we make better use of RAM. Then - * processDocument is called on that ThreadState without - * synchronization (most of the "heavy lifting" is in this - * call). Finally the synchronized "finishDocument" is - * called to flush changes to the directory. - * - * Each ThreadState instance has its own Posting hash. Once - * we're using too much RAM, we flush all Posting hashes to - * a segment by merging the docIDs in the posting lists for - * the same term across multiple thread states (see + * + * Multiple threads are allowed into addDocument at once. There is an initial + * synchronized call to getThreadState which allocates a ThreadState for this + * thread. The same thread will get the same ThreadState over time (thread + * affinity) so that if there are consistent patterns (for example each thread + * is indexing a different content source) then we make better use of RAM. Then + * processDocument is called on that ThreadState without synchronization (most + * of the "heavy lifting" is in this call). Finally the synchronized + * "finishDocument" is called to flush changes to the directory. + * + * Each ThreadState instance has its own Posting hash. Once we're using too much + * RAM, we flush all Posting hashes to a segment by merging the docIDs in the + * posting lists for the same term across multiple thread states (see * writeSegment and appendPostings). - * - * When flush is called by IndexWriter, or, we flush - * internally when autoCommit=false, we forcefully idle all - * threads and flush only once they are all idle. This - * means you can call flush with a given thread even while - * other threads are actively adding/deleting documents. - * - * + * + * When flush is called by IndexWriter, or, we flush internally when + * autoCommit=false, we forcefully idle all threads and flush only once they are + * all idle. This means you can call flush with a given thread even while other + * threads are actively adding/deleting documents. + * + * * Exceptions: - * - * Because this class directly updates in-memory posting - * lists, and flushes stored fields and term vectors - * directly to files in the directory, there are certain - * limited times when an exception can corrupt this state. - * For example, a disk full while flushing stored fields - * leaves this file in a corrupt state. Or, an OOM - * exception while appending to the in-memory posting lists - * can corrupt that posting list. We call such exceptions - * "aborting exceptions". In these cases we must call - * abort() to discard all docs added since the last flush. - * - * All other exceptions ("non-aborting exceptions") can - * still partially update the index structures. These - * updates are consistent, but, they represent only a part - * of the document seen up until the exception was hit. - * When this happens, we immediately mark the document as - * deleted so that the document is always atomically ("all - * or none") added to the index. + * + * Because this class directly updates in-memory posting lists, and flushes + * stored fields and term vectors directly to files in the directory, there are + * certain limited times when an exception can corrupt this state. For example, + * a disk full while flushing stored fields leaves this file in a corrupt state. + * Or, an OOM exception while appending to the in-memory posting lists can + * corrupt that posting list. We call such exceptions "aborting exceptions". In + * these cases we must call abort() to discard all docs added since the last + * flush. + * + * All other exceptions ("non-aborting exceptions") can still partially update + * the index structures. These updates are consistent, but, they represent only + * a part of the document seen up until the exception was hit. When this + * happens, we immediately mark the document as deleted so that the document is + * always atomically ("all or none") added to the index. */ final class DocumentsWriter { - IndexWriter writer; - Directory directory; + IndexWriter writer; + Directory directory; - FieldInfos fieldInfos = new FieldInfos(); // All fields we've seen - IndexOutput tvx, tvf, tvd; // To write term vectors - FieldsWriter fieldsWriter; // To write stored fields + FieldInfos fieldInfos = new FieldInfos(); // All fields we've seen + IndexOutput tvx, tvf, tvd; // To write term vectors + FieldsWriter fieldsWriter; // To write stored fields - String segment; // Current segment we are working on - String docStoreSegment; // Current doc-store segment we are writing - private int docStoreOffset; // Current starting doc-store offset of current segment + String segment; // Current segment we are working on + String docStoreSegment; // Current doc-store segment we are writing + private int docStoreOffset; // Current starting doc-store offset of current + // segment - private int nextDocID; // Next docID to be added - private int numDocsInRAM; // # docs buffered in RAM - int numDocsInStore; // # docs written to doc stores - private int nextWriteDocID; // Next docID to be written + private int nextDocID; // Next docID to be added + private int numDocsInRAM; // # docs buffered in RAM + int numDocsInStore; // # docs written to doc stores + private int nextWriteDocID; // Next docID to be written - // Max # ThreadState instances; if there are more threads - // than this they share ThreadStates - private final static int MAX_THREAD_STATE = 5; - private DocumentsWriterThreadState[] threadStates = new DocumentsWriterThreadState[0]; - private final HashMap threadBindings = new HashMap(); - private int numWaiting; - private final DocumentsWriterThreadState[] waitingThreadStates = new DocumentsWriterThreadState[MAX_THREAD_STATE]; - private int pauseThreads; // Non-zero when we need all threads to - // pause (eg to flush) - boolean flushPending; // True when a thread has decided to flush - boolean bufferIsFull; // True when it's time to write segment - private int abortCount; // Non-zero while abort is pending or running + // Max # ThreadState instances; if there are more threads + // than this they share ThreadStates + private final static int MAX_THREAD_STATE = 5; + private DocumentsWriterThreadState[] threadStates = new DocumentsWriterThreadState[0]; + private final HashMap threadBindings = new HashMap(); + private int numWaiting; + private final DocumentsWriterThreadState[] waitingThreadStates = new DocumentsWriterThreadState[MAX_THREAD_STATE]; + private int pauseThreads; // Non-zero when we need all threads to + // pause (eg to flush) + boolean flushPending; // True when a thread has decided to flush + boolean bufferIsFull; // True when it's time to write segment + private int abortCount; // Non-zero while abort is pending or running - PrintStream infoStream; + PrintStream infoStream; - boolean hasNorms; // Whether any norms were seen since last flush + boolean hasNorms; // Whether any norms were seen since last flush - List newFiles; + List newFiles; - // Deletes done after the last flush; these are discarded - // on abort - private BufferedDeletes deletesInRAM = new BufferedDeletes(); + // Deletes done after the last flush; these are discarded + // on abort + private BufferedDeletes deletesInRAM = new BufferedDeletes(); - // Deletes done before the last flush; these are still - // kept on abort - private BufferedDeletes deletesFlushed = new BufferedDeletes(); + // Deletes done before the last flush; these are still + // kept on abort + private BufferedDeletes deletesFlushed = new BufferedDeletes(); - // The max number of delete terms that can be buffered before - // they must be flushed to disk. - private int maxBufferedDeleteTerms = IndexWriter.DEFAULT_MAX_BUFFERED_DELETE_TERMS; + // The max number of delete terms that can be buffered before + // they must be flushed to disk. + private int maxBufferedDeleteTerms = IndexWriter.DEFAULT_MAX_BUFFERED_DELETE_TERMS; - // How much RAM we can use before flushing. This is 0 if - // we are flushing by doc count instead. - private long ramBufferSize = (long) (IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024); + // How much RAM we can use before flushing. This is 0 if + // we are flushing by doc count instead. + private long ramBufferSize = (long) (IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB * 1024 * 1024); - // Flush @ this number of docs. If rarmBufferSize is - // non-zero we will flush by RAM usage instead. - private int maxBufferedDocs = IndexWriter.DEFAULT_MAX_BUFFERED_DOCS; + // Flush @ this number of docs. If rarmBufferSize is + // non-zero we will flush by RAM usage instead. + private int maxBufferedDocs = IndexWriter.DEFAULT_MAX_BUFFERED_DOCS; - private int flushedDocCount; // How many docs already flushed to index + private int flushedDocCount; // How many docs already flushed to index + + // Used for append posting term docs + private IntArrayList termDocsList = new IntArrayList(50, 1.25); + + synchronized void updateFlushedDocCount(int n) { + flushedDocCount += n; + } - synchronized void updateFlushedDocCount(int n) { - flushedDocCount += n; - } - synchronized int getFlushedDocCount() { - return flushedDocCount; - } - synchronized void setFlushedDocCount(int n) { - flushedDocCount = n; - } + synchronized int getFlushedDocCount() { + return flushedDocCount; + } - private boolean closed; + synchronized void setFlushedDocCount(int n) { + flushedDocCount = n; + } - // Coarse estimates used to measure RAM usage of buffered deletes - private static int OBJECT_HEADER_BYTES = 8; + private boolean closed; - BufferedNorms[] norms = new BufferedNorms[0]; // Holds norms until we flush + // Coarse estimates used to measure RAM usage of buffered deletes + private static int OBJECT_HEADER_BYTES = 8; - DocumentsWriter(Directory directory, IndexWriter writer) throws IOException { - this.directory = directory; - this.writer = writer; - flushedDocCount = writer.docCount(); - postingsFreeList = new Posting[0]; - } + BufferedNorms[] norms = new BufferedNorms[0]; // Holds norms until we flush - /** If non-null, various details of indexing are printed - * here. */ - void setInfoStream(PrintStream infoStream) { - this.infoStream = infoStream; - } + DocumentsWriter(Directory directory, IndexWriter writer) throws IOException { + this.directory = directory; + this.writer = writer; + flushedDocCount = writer.docCount(); + postingsFreeList = new Posting[0]; + } - /** Set how much RAM we can use before flushing. */ - void setRAMBufferSizeMB(double mb) { - if (mb == IndexWriter.DISABLE_AUTO_FLUSH) { - ramBufferSize = IndexWriter.DISABLE_AUTO_FLUSH; - } else { - ramBufferSize = (long) (mb*1024*1024); - } - } + /** + * If non-null, various details of indexing are printed here. + */ + void setInfoStream(PrintStream infoStream) { + this.infoStream = infoStream; + } - double getRAMBufferSizeMB() { - if (ramBufferSize == IndexWriter.DISABLE_AUTO_FLUSH) { - return ramBufferSize; - } else { - return ramBufferSize/1024./1024.; - } - } + /** Set how much RAM we can use before flushing. */ + void setRAMBufferSizeMB(double mb) { + if (mb == IndexWriter.DISABLE_AUTO_FLUSH) { + ramBufferSize = IndexWriter.DISABLE_AUTO_FLUSH; + } else { + ramBufferSize = (long) (mb * 1024 * 1024); + } + } - /** Set max buffered docs, which means we will flush by - * doc count instead of by RAM usage. */ - void setMaxBufferedDocs(int count) { - maxBufferedDocs = count; - } + double getRAMBufferSizeMB() { + if (ramBufferSize == IndexWriter.DISABLE_AUTO_FLUSH) { + return ramBufferSize; + } else { + return ramBufferSize / 1024. / 1024.; + } + } - int getMaxBufferedDocs() { - return maxBufferedDocs; - } + /** + * Set max buffered docs, which means we will flush by doc count instead of by + * RAM usage. + */ + void setMaxBufferedDocs(int count) { + maxBufferedDocs = count; + } - /** Get current segment name we are writing. */ - String getSegment() { - return segment; - } + int getMaxBufferedDocs() { + return maxBufferedDocs; + } - /** Returns how many docs are currently buffered in RAM. */ - int getNumDocsInRAM() { - return numDocsInRAM; - } + /** Get current segment name we are writing. */ + String getSegment() { + return segment; + } - /** Returns the current doc store segment we are writing - * to. This will be the same as segment when autoCommit - * * is true. */ - String getDocStoreSegment() { - return docStoreSegment; - } + /** Returns how many docs are currently buffered in RAM. */ + int getNumDocsInRAM() { + return numDocsInRAM; + } - /** Returns the doc offset into the shared doc store for - * the current buffered docs. */ - int getDocStoreOffset() { - return docStoreOffset; - } + /** + * Returns the current doc store segment we are writing to. This will be the + * same as segment when autoCommit * is true. + */ + String getDocStoreSegment() { + return docStoreSegment; + } - /** Closes the current open doc stores an returns the doc - * store segment name. This returns null if there are * - * no buffered documents. */ - String closeDocStore() throws IOException { + /** + * Returns the doc offset into the shared doc store for the current buffered + * docs. + */ + int getDocStoreOffset() { + return docStoreOffset; + } - assert allThreadsIdle(); + /** + * Closes the current open doc stores an returns the doc store segment name. + * This returns null if there are * no buffered documents. + */ + String closeDocStore() throws IOException { - List flushedFiles = files(); + assert allThreadsIdle(); - if (infoStream != null) - message("closeDocStore: " + flushedFiles.size() + " files to flush to segment " + docStoreSegment + " numDocs=" + numDocsInStore); + List flushedFiles = files(); - if (flushedFiles.size() > 0) { - files = null; + if (infoStream != null) + message("closeDocStore: " + flushedFiles.size() + + " files to flush to segment " + docStoreSegment + " numDocs=" + + numDocsInStore); - if (tvx != null) { - // At least one doc in this run had term vectors enabled - assert docStoreSegment != null; - tvx.close(); - tvf.close(); - tvd.close(); - tvx = null; - assert 4+numDocsInStore*16 == directory.fileLength(docStoreSegment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION): - "after flush: tvx size mismatch: " + numDocsInStore + " docs vs " + directory.fileLength(docStoreSegment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION) + " length in bytes of " + docStoreSegment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION; - } + if (flushedFiles.size() > 0) { + files = null; - if (fieldsWriter != null) { - assert docStoreSegment != null; - fieldsWriter.close(); - fieldsWriter = null; - assert 4+numDocsInStore*8 == directory.fileLength(docStoreSegment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION): - "after flush: fdx size mismatch: " + numDocsInStore + " docs vs " + directory.fileLength(docStoreSegment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION) + " length in bytes of " + docStoreSegment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION; - } + if (tvx != null) { + // At least one doc in this run had term vectors enabled + assert docStoreSegment != null; + tvx.close(); + tvf.close(); + tvd.close(); + tvx = null; + assert 4 + numDocsInStore * 16 == directory.fileLength(docStoreSegment + + "." + IndexFileNames.VECTORS_INDEX_EXTENSION) : "after flush: tvx size mismatch: " + + numDocsInStore + + " docs vs " + + directory.fileLength(docStoreSegment + "." + + IndexFileNames.VECTORS_INDEX_EXTENSION) + + " length in bytes of " + + docStoreSegment + + "." + + IndexFileNames.VECTORS_INDEX_EXTENSION; + } - String s = docStoreSegment; - docStoreSegment = null; - docStoreOffset = 0; - numDocsInStore = 0; - return s; - } else { - return null; - } - } + if (fieldsWriter != null) { + assert docStoreSegment != null; + fieldsWriter.close(); + fieldsWriter = null; + assert 4 + numDocsInStore * 8 == directory.fileLength(docStoreSegment + + "." + IndexFileNames.FIELDS_INDEX_EXTENSION) : "after flush: fdx size mismatch: " + + numDocsInStore + + " docs vs " + + directory.fileLength(docStoreSegment + "." + + IndexFileNames.FIELDS_INDEX_EXTENSION) + + " length in bytes of " + + docStoreSegment + + "." + + IndexFileNames.FIELDS_INDEX_EXTENSION; + } - List files = null; // Cached list of files we've created - private List abortedFiles = null; // List of files that were written before last abort() + String s = docStoreSegment; + docStoreSegment = null; + docStoreOffset = 0; + numDocsInStore = 0; + return s; + } else { + return null; + } + } - List abortedFiles() { - return abortedFiles; - } + List files = null; // Cached list of files we've created + private List abortedFiles = null; // List of files that were written before - void message(String message) { - writer.message("DW: " + message); - } + // last abort() - /* Returns list of files in use by this instance, - * including any flushed segments. */ - synchronized List files() { + List abortedFiles() { + return abortedFiles; + } - if (files != null) - return files; + void message(String message) { + writer.message("DW: " + message); + } - files = new ArrayList(); + /* + * Returns list of files in use by this instance, including any flushed + * segments. + */ + synchronized List files() { - // Stored fields: - if (fieldsWriter != null) { - assert docStoreSegment != null; - files.add(docStoreSegment + "." + IndexFileNames.FIELDS_EXTENSION); - files.add(docStoreSegment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION); - } + if (files != null) + return files; - // Vectors: - if (tvx != null) { - assert docStoreSegment != null; - files.add(docStoreSegment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION); - files.add(docStoreSegment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION); - files.add(docStoreSegment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); - } + files = new ArrayList(); - return files; - } + // Stored fields: + if (fieldsWriter != null) { + assert docStoreSegment != null; + files.add(docStoreSegment + "." + IndexFileNames.FIELDS_EXTENSION); + files.add(docStoreSegment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION); + } - synchronized void setAborting() { - abortCount++; - } + // Vectors: + if (tvx != null) { + assert docStoreSegment != null; + files.add(docStoreSegment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION); + files + .add(docStoreSegment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION); + files.add(docStoreSegment + "." + + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); + } - /** Called if we hit an exception when adding docs, - * flushing, etc. This resets our state, discarding any - * docs added since last flush. If ae is non-null, it - * contains the root cause exception (which we re-throw - * after we are done aborting). */ - synchronized void abort(AbortException ae) throws IOException { + return files; + } - // Anywhere that throws an AbortException must first - // mark aborting to make sure while the exception is - // unwinding the un-synchronized stack, no thread grabs - // the corrupt ThreadState that hit the aborting - // exception: - assert ae == null || abortCount>0; + synchronized void setAborting() { + abortCount++; + } - try { + /** + * Called if we hit an exception when adding docs, flushing, etc. This resets + * our state, discarding any docs added since last flush. If ae is non-null, + * it contains the root cause exception (which we re-throw after we are done + * aborting). + */ + synchronized void abort(AbortException ae) throws IOException { - if (infoStream != null) - message("docWriter: now abort"); + // Anywhere that throws an AbortException must first + // mark aborting to make sure while the exception is + // unwinding the un-synchronized stack, no thread grabs + // the corrupt ThreadState that hit the aborting + // exception: + assert ae == null || abortCount > 0; - // Forcefully remove waiting ThreadStates from line - for(int i=0;i 0; - } + // If we have a root cause exception, re-throw it now: + if (ae != null) { + Throwable t = ae.getCause(); + if (t instanceof IOException) + throw (IOException) t; + else if (t instanceof RuntimeException) + throw (RuntimeException) t; + else if (t instanceof Error) + throw (Error) t; + else + // Should not get here + assert false : "unknown exception: " + t; + } + } finally { + if (ae != null) + abortCount--; + notifyAll(); + } + } - synchronized void resumeAllThreads() { - pauseThreads--; - assert pauseThreads >= 0; - if (0 == pauseThreads) - notifyAll(); - } + /** Reset after a flush */ + private void resetPostingsData() throws IOException { + // All ThreadStates should be idle when we are called + assert allThreadsIdle(); + threadBindings.clear(); + segment = null; + numDocsInRAM = 0; + nextDocID = 0; + nextWriteDocID = 0; + files = null; + balanceRAM(); + bufferIsFull = false; + flushPending = false; + for (int i = 0; i < threadStates.length; i++) { + threadStates[i].numThreads = 0; + threadStates[i].resetPostings(); + } + numBytesUsed = 0; + } - private synchronized boolean allThreadsIdle() { - for(int i=0;i 0; + } - /** Flush all pending docs to a new segment */ - synchronized int flush(boolean closeDocStore) throws IOException { + synchronized void resumeAllThreads() { + pauseThreads--; + assert pauseThreads >= 0; + if (0 == pauseThreads) + notifyAll(); + } - assert allThreadsIdle(); + private synchronized boolean allThreadsIdle() { + for (int i = 0; i < threadStates.length; i++) + if (!threadStates[i].isIdle) + return false; + return true; + } - if (segment == null) - // In case we are asked to flush an empty segment - segment = writer.newSegmentName(); + /** Flush all pending docs to a new segment */ + synchronized int flush(boolean closeDocStore) throws IOException { - newFiles = new ArrayList(); + assert allThreadsIdle(); - docStoreOffset = numDocsInStore; + if (segment == null) + // In case we are asked to flush an empty segment + segment = writer.newSegmentName(); - int docCount; + newFiles = new ArrayList(); - assert numDocsInRAM > 0; + docStoreOffset = numDocsInStore; - if (infoStream != null) - message("flush postings as segment " + segment + " numDocs=" + numDocsInRAM); - - boolean success = false; + int docCount; - try { + assert numDocsInRAM > 0; - if (closeDocStore) { - assert docStoreSegment != null; - assert docStoreSegment.equals(segment); - newFiles.addAll(files()); - closeDocStore(); - } - - fieldInfos.write(directory, segment + ".fnm"); + if (infoStream != null) + message("flush postings as segment " + segment + " numDocs=" + + numDocsInRAM); - docCount = numDocsInRAM; + boolean success = false; - newFiles.addAll(writeSegment()); + try { - flushedDocCount += docCount; + if (closeDocStore) { + assert docStoreSegment != null; + assert docStoreSegment.equals(segment); + newFiles.addAll(files()); + closeDocStore(); + } - success = true; + fieldInfos.write(directory, segment + ".fnm"); - } finally { - if (!success) - abort(null); - } + docCount = numDocsInRAM; - return docCount; - } + newFiles.addAll(writeSegment()); - /** Build compound file for the segment we just flushed */ - void createCompoundFile(String segment) throws IOException - { - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION); - final int size = newFiles.size(); - for(int i=0;i 0) - allFields.add(fp); - } - } + assert nextDocID == numDocsInRAM; - // Sort by field name - Collections.sort(allFields); - final int numAllFields = allFields.size(); + final String segmentName; - skipListWriter = new DefaultSkipListWriter(termsOut.skipInterval, - termsOut.maxSkipLevels, - numDocsInRAM, freqOut, proxOut); + segmentName = segment; - int start = 0; - while(start < numAllFields) { + TermInfosWriter termsOut = new TermInfosWriter(directory, segmentName, + fieldInfos, writer.getTermIndexInterval()); - final String fieldName = ((DocumentsWriterFieldData) allFields.get(start)).fieldInfo.name; + IndexOutput freqOut = directory.createOutput(segmentName + ".frq"); + IndexOutput proxOut = directory.createOutput(segmentName + ".prx"); - int end = start+1; - while(end < numAllFields && ((DocumentsWriterFieldData) allFields.get(end)).fieldInfo.name.equals(fieldName)) - end++; - - DocumentsWriterFieldData[] fields = new DocumentsWriterFieldData[end-start]; - for(int i=start;i 0) + allFields.add(fp); + } + } - // If this field has postings then add them to the - // segment - appendPostings(fields, termsOut, freqOut, proxOut); + // Sort by field name + Collections.sort(allFields); + final int numAllFields = allFields.size(); - for(int i=0;i 1.5*postingsFreeCount) { - int newSize = postingsFreeList.length; - while(newSize > 1.25*postingsFreeCount) { - newSize = (int) (newSize*0.8); - } - Posting[] newArray = new Posting[newSize]; - System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount); - postingsFreeList = newArray; - } + // If this field has postings then add them to the + // segment + appendPostings(fields, termsOut, freqOut, proxOut); - return flushedFiles; - } + for (int i = 0; i < fields.length; i++) + fields[i].resetPostingArrays(); - synchronized void pushDeletes() { - deletesFlushed.update(deletesInRAM); - } + start = end; + } - /** Returns the name of the file with this extension, on - * the current segment we are working on. */ - private String segmentFileName(String extension) { - return segment + "." + extension; - } + freqOut.close(); + proxOut.close(); + termsOut.close(); - private static int compareText(final char[] text1, int pos1, final char[] text2, int pos2) { - while(true) { - final char c1 = text1[pos1++]; - final char c2 = text2[pos2++]; - if (c1 != c2) { - if (0xffff == c2) - return 1; - else if (0xffff == c1) - return -1; - else - return c1-c2; - } else if (0xffff == c1) - return 0; - } - } + // Record all files we have flushed + List flushedFiles = new ArrayList(); + flushedFiles.add(segmentFileName(IndexFileNames.FIELD_INFOS_EXTENSION)); + flushedFiles.add(segmentFileName(IndexFileNames.FREQ_EXTENSION)); + flushedFiles.add(segmentFileName(IndexFileNames.PROX_EXTENSION)); + flushedFiles.add(segmentFileName(IndexFileNames.TERMS_EXTENSION)); + flushedFiles.add(segmentFileName(IndexFileNames.TERMS_INDEX_EXTENSION)); - private final TermInfo termInfo = new TermInfo(); // minimize consing + if (hasNorms) { + writeNorms(segmentName, numDocsInRAM); + flushedFiles.add(segmentFileName(IndexFileNames.NORMS_EXTENSION)); + } - final UnicodeUtil.UTF8Result termsUTF8 = new UnicodeUtil.UTF8Result(); + if (infoStream != null) { + final long newSegmentSize = segmentSize(segmentName); + String message = " oldRAMSize=" + numBytesUsed + " newFlushedSize=" + + newSegmentSize + " docs/MB=" + + nf.format(numDocsInRAM / (newSegmentSize / 1024. / 1024.)) + + " new/old=" + nf.format(100.0 * newSegmentSize / numBytesUsed) + + "%"; + message(message); + } - /* Walk through all unique text tokens (Posting - * instances) found in this field and serialize them - * into a single RAM segment. */ - void appendPostings(DocumentsWriterFieldData[] fields, - TermInfosWriter termsOut, - IndexOutput freqOut, - IndexOutput proxOut) - throws CorruptIndexException, IOException { + resetPostingsData(); - final int fieldNumber = fields[0].fieldInfo.number; - int numFields = fields.length; + // Maybe downsize postingsFreeList array + if (postingsFreeList.length > 1.5 * postingsFreeCount) { + int newSize = postingsFreeList.length; + while (newSize > 1.25 * postingsFreeCount) { + newSize = (int) (newSize * 0.8); + } + Posting[] newArray = new Posting[newSize]; + System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount); + postingsFreeList = newArray; + } - final DocumentsWriterFieldMergeState[] mergeStates = new DocumentsWriterFieldMergeState[numFields]; + return flushedFiles; + } - for(int i=0;i 0) { + /* + * Walk through all unique text tokens (Posting instances) found in this field + * and serialize them into a single RAM segment. + */ + void appendPostings(DocumentsWriterFieldData[] fields, + TermInfosWriter termsOut, IndexOutput freqOut, IndexOutput proxOut) + throws CorruptIndexException, IOException { - // Get the next term to merge - termStates[0] = mergeStates[0]; - int numToMerge = 1; + final int fieldNumber = fields[0].fieldInfo.number; + int numFields = fields.length; - for(int i=1;i 0) { - // Now termStates has numToMerge FieldMergeStates - // which all share the same term. Now we must - // interleave the docID streams. - while(numToMerge > 0) { - - if ((++df % skipInterval) == 0) { - skipListWriter.setSkipData(lastDoc, currentFieldStorePayloads, lastPayloadLength); - skipListWriter.bufferSkip(df); - } + // Get the next term to merge + termStates[0] = mergeStates[0]; + int numToMerge = 1; - DocumentsWriterFieldMergeState minState = termStates[0]; - for(int i=1;i lastDoc || df == 1; + int df = 0; + int lastPayloadLength = -1; - final int newDocCode = (doc-lastDoc)<<1; - lastDoc = doc; + int lastDoc = 0; - final ByteSliceReader prox = minState.prox; + final char[] text = termStates[0].text; + final int start = termStates[0].textOffset; - // Carefully copy over the prox + payload info, - // changing the format to match Lucene's segment - // format. - for(int j=0;j 0) - copyBytes(prox, proxOut, payloadLength); - } else { - assert 0 == (code & 1); - proxOut.writeVInt(code>>1); - } - } + long freqPointer = freqOut.getFilePointer(); + long proxPointer = proxOut.getFilePointer(); - if (1 == termDocFreq) { - freqOut.writeVInt(newDocCode|1); - } else { - freqOut.writeVInt(newDocCode); - freqOut.writeVInt(termDocFreq); - } + skipListWriter.resetSkip(); - if (!minState.nextDoc()) { + boolean doTermDocs = termStates[0].field.doTermDocs; + termDocsList.clear(); + // Now termStates has numToMerge FieldMergeStates + // which all share the same term. Now we must + // interleave the docID streams. + while (numToMerge > 0) { - // Remove from termStates - int upto = 0; - for(int i=0;i 0; + final int termDocFreq = minState.termFreq; - // Done merging this term + assert doc < numDocsInRAM; + assert doc > lastDoc || df == 1; - long skipPointer = skipListWriter.writeSkip(freqOut); + final int newDocCode = (doc - lastDoc) << 1; + lastDoc = doc; - // Write term - termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer)); + final ByteSliceReader prox = minState.prox; - // TODO: we could do this incrementally - UnicodeUtil.UTF16toUTF8(text, start, termsUTF8); + // Carefully copy over the prox + payload info, + // changing the format to match Lucene's segment + // format. + for (int j = 0; j < termDocFreq; j++) { + final int code = prox.readVInt(); + if (currentFieldStorePayloads) { + final int payloadLength; + if ((code & 1) != 0) { + // This position has a payload + payloadLength = prox.readVInt(); + } else + payloadLength = 0; + if (payloadLength != lastPayloadLength) { + proxOut.writeVInt(code | 1); + proxOut.writeVInt(payloadLength); + lastPayloadLength = payloadLength; + } else + proxOut.writeVInt(code & (~1)); + if (payloadLength > 0) + copyBytes(prox, proxOut, payloadLength); + } else { + assert 0 == (code & 1); + proxOut.writeVInt(code >> 1); + } + } - // TODO: we could save O(n) re-scan of the term by - // computing the shared prefix with the last term - // while during the UTF8 encoding - termsOut.add(fieldNumber, - termsUTF8.result, - termsUTF8.length, - termInfo); - } - } + if (1 == termDocFreq) { + freqOut.writeVInt(newDocCode | 1); + } else { + freqOut.writeVInt(newDocCode); + freqOut.writeVInt(termDocFreq); + } - synchronized void close() { - closed = true; - notifyAll(); - } + if (!minState.nextDoc()) { - /** Returns a free (idle) ThreadState that may be used for - * indexing this one document. This call also pauses if a - * flush is pending. If delTerm is non-null then we - * buffer this deleted term after the thread state has - * been acquired. */ - synchronized DocumentsWriterThreadState getThreadState(Document doc, Term delTerm) throws IOException { + // Remove from termStates + int upto = 0; + for (int i = 0; i < numToMerge; i++) + if (termStates[i] != minState) + termStates[upto++] = termStates[i]; + numToMerge--; + assert upto == numToMerge; - // First, find a thread state. If this thread already - // has affinity to a specific ThreadState, use that one - // again. - DocumentsWriterThreadState state = (DocumentsWriterThreadState) threadBindings.get(Thread.currentThread()); - if (state == null) { - // First time this thread has called us since last flush - DocumentsWriterThreadState minThreadState = null; - for(int i=0;i 0) - System.arraycopy(threadStates, 0, newArray, 0, threadStates.length); - state = newArray[threadStates.length] = new DocumentsWriterThreadState(this); - threadStates = newArray; - } - threadBindings.put(Thread.currentThread(), state); - } + // Advance this state to the next term - // Next, wait until my thread state is idle (in case - // it's shared with other threads) and for threads to - // not be paused nor a flush pending: - waitReady(state); + if (!minState.nextTerm()) { + // OK, no more terms, so remove from mergeStates + // as well + upto = 0; + for (int i = 0; i < numFields; i++) + if (mergeStates[i] != minState) + mergeStates[upto++] = mergeStates[i]; + numFields--; + assert upto == numFields; + } + } + } - if (segment == null) - segment = writer.newSegmentName(); + assert df > 0; - state.isIdle = false; + // Done merging this term - try { - boolean success = false; - try { - state.init(doc, nextDocID); - if (delTerm != null) { - addDeleteTerm(delTerm, state.docID); - state.doFlushAfter = timeToFlushDeletes(); - } - // Only increment nextDocID & numDocsInRAM on successful init - nextDocID++; - numDocsInRAM++; + long skipPointer = skipListWriter.writeSkip(freqOut); + // create the term docs array if needed + int[] docs = null; + + if (termStates[0].field.doTermDocs) { + docs = termDocsList.toIntArray(); + } + + // Write term + termInfo.set(df, freqPointer, proxPointer, + (int) (skipPointer - freqPointer), docs); - // We must at this point commit to flushing to ensure we - // always get N docs when we flush by doc count, even if - // > 1 thread is adding documents: - if (!flushPending && maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH - && numDocsInRAM >= maxBufferedDocs) { - flushPending = true; - state.doFlushAfter = true; - } + // TODO: we could do this incrementally + UnicodeUtil.UTF16toUTF8(text, start, termsUTF8); - success = true; - } finally { - if (!success) { - // Forcefully idle this ThreadState: - state.isIdle = true; - notifyAll(); - if (state.doFlushAfter) { - state.doFlushAfter = false; - flushPending = false; - } - } - } - } catch (AbortException ae) { - abort(ae); - } + // TODO: we could save O(n) re-scan of the term by + // computing the shared prefix with the last term + // while during the UTF8 encoding + termsOut.add(fieldNumber, termsUTF8.result, termsUTF8.length, termInfo); + } + } - return state; - } + synchronized void close() { + closed = true; + notifyAll(); + } - /** Returns true if the caller (IndexWriter) should now - * flush. */ - boolean addDocument(Document doc, Analyzer analyzer) - throws CorruptIndexException, IOException { - return updateDocument(doc, analyzer, null); - } + /** + * Returns a free (idle) ThreadState that may be used for indexing this one + * document. This call also pauses if a flush is pending. If delTerm is + * non-null then we buffer this deleted term after the thread state has been + * acquired. + */ + synchronized DocumentsWriterThreadState getThreadState(Document doc, + Term delTerm) throws IOException { - boolean updateDocument(Term t, Document doc, Analyzer analyzer) - throws CorruptIndexException, IOException { - return updateDocument(doc, analyzer, t); - } + // First, find a thread state. If this thread already + // has affinity to a specific ThreadState, use that one + // again. + DocumentsWriterThreadState state = (DocumentsWriterThreadState) threadBindings + .get(Thread.currentThread()); + if (state == null) { + // First time this thread has called us since last flush + DocumentsWriterThreadState minThreadState = null; + for (int i = 0; i < threadStates.length; i++) { + DocumentsWriterThreadState ts = threadStates[i]; + if (minThreadState == null || ts.numThreads < minThreadState.numThreads) + minThreadState = ts; + } + if (minThreadState != null + && (minThreadState.numThreads == 0 || threadStates.length == MAX_THREAD_STATE)) { + state = minThreadState; + state.numThreads++; + } else { + // Just create a new "private" thread state + DocumentsWriterThreadState[] newArray = new DocumentsWriterThreadState[1 + threadStates.length]; + if (threadStates.length > 0) + System.arraycopy(threadStates, 0, newArray, 0, threadStates.length); + state = newArray[threadStates.length] = new DocumentsWriterThreadState( + this); + threadStates = newArray; + } + threadBindings.put(Thread.currentThread(), state); + } - boolean updateDocument(Document doc, Analyzer analyzer, Term delTerm) - throws CorruptIndexException, IOException { + // Next, wait until my thread state is idle (in case + // it's shared with other threads) and for threads to + // not be paused nor a flush pending: + waitReady(state); - // This call is synchronized but fast - final DocumentsWriterThreadState state = getThreadState(doc, delTerm); - try { - boolean success = false; - try { - try { - // This call is not synchronized and does all the work - state.processDocument(analyzer); - } finally { - // Note that we must call finishDocument even on - // exception, because for a non-aborting - // exception, a portion of the document has been - // indexed (and its ID is marked for deletion), so - // all index files must be updated to record this - // document. This call is synchronized but fast. - finishDocument(state); - } - success = true; - } finally { - if (!success) { - synchronized(this) { + if (segment == null) + segment = writer.newSegmentName(); - // If this thread state had decided to flush, we - // must clear it so another thread can flush - if (state.doFlushAfter) { - state.doFlushAfter = false; - flushPending = false; - notifyAll(); - } + state.isIdle = false; - // Immediately mark this document as deleted - // since likely it was partially added. This - // keeps indexing as "all or none" (atomic) when - // adding a document: - addDeleteDocID(state.docID); - } - } - } - } catch (AbortException ae) { - abort(ae); - } + try { + boolean success = false; + try { + state.init(doc, nextDocID); + if (delTerm != null) { + addDeleteTerm(delTerm, state.docID); + state.doFlushAfter = timeToFlushDeletes(); + } + // Only increment nextDocID & numDocsInRAM on successful init + nextDocID++; + numDocsInRAM++; - return state.doFlushAfter || timeToFlushDeletes(); - } + // We must at this point commit to flushing to ensure we + // always get N docs when we flush by doc count, even if + // > 1 thread is adding documents: + if (!flushPending && maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH + && numDocsInRAM >= maxBufferedDocs) { + flushPending = true; + state.doFlushAfter = true; + } - // for testing - synchronized int getNumBufferedDeleteTerms() { - return deletesInRAM.numTerms; - } + success = true; + } finally { + if (!success) { + // Forcefully idle this ThreadState: + state.isIdle = true; + notifyAll(); + if (state.doFlushAfter) { + state.doFlushAfter = false; + flushPending = false; + } + } + } + } catch (AbortException ae) { + abort(ae); + } - // for testing - synchronized HashMap getBufferedDeleteTerms() { - return deletesInRAM.terms; - } + return state; + } - /** Called whenever a merge has completed and the merged segments had deletions */ - synchronized void remapDeletes(SegmentInfos infos, int[][] docMaps, int[] delCounts, MergePolicy.OneMerge merge, int mergeDocCount) { - if (docMaps == null) - // The merged segments had no deletes so docIDs did not change and we have nothing to do - return; - MergeDocIDRemapper mapper = new MergeDocIDRemapper(infos, docMaps, delCounts, merge, mergeDocCount); - deletesInRAM.remap(mapper, infos, docMaps, delCounts, merge, mergeDocCount); - deletesFlushed.remap(mapper, infos, docMaps, delCounts, merge, mergeDocCount); - flushedDocCount -= mapper.docShift; - } + /** + * Returns true if the caller (IndexWriter) should now flush. + */ + boolean addDocument(Document doc, Analyzer analyzer) + throws CorruptIndexException, IOException { + return updateDocument(doc, analyzer, null); + } - synchronized private void waitReady(DocumentsWriterThreadState state) { - while(!closed && ((state != null && !state.isIdle) || pauseThreads != 0 || flushPending || abortCount > 0)) - try { - wait(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + boolean updateDocument(Term t, Document doc, Analyzer analyzer) + throws CorruptIndexException, IOException { + return updateDocument(doc, analyzer, t); + } - if (closed) - throw new AlreadyClosedException("this IndexWriter is closed"); - } + boolean updateDocument(Document doc, Analyzer analyzer, Term delTerm) + throws CorruptIndexException, IOException { - synchronized boolean bufferDeleteTerms(Term[] terms) throws IOException { - waitReady(null); - for (int i = 0; i < terms.length; i++) - addDeleteTerm(terms[i], numDocsInRAM); - return timeToFlushDeletes(); - } + // This call is synchronized but fast + final DocumentsWriterThreadState state = getThreadState(doc, delTerm); + try { + boolean success = false; + try { + try { + // This call is not synchronized and does all the work + state.processDocument(analyzer); + } finally { + // Note that we must call finishDocument even on + // exception, because for a non-aborting + // exception, a portion of the document has been + // indexed (and its ID is marked for deletion), so + // all index files must be updated to record this + // document. This call is synchronized but fast. + finishDocument(state); + } + success = true; + } finally { + if (!success) { + synchronized (this) { - synchronized boolean bufferDeleteTerm(Term term) throws IOException { - waitReady(null); - addDeleteTerm(term, numDocsInRAM); - return timeToFlushDeletes(); - } + // If this thread state had decided to flush, we + // must clear it so another thread can flush + if (state.doFlushAfter) { + state.doFlushAfter = false; + flushPending = false; + notifyAll(); + } - synchronized boolean bufferDeleteQueries(Query[] queries) throws IOException { - waitReady(null); - for (int i = 0; i < queries.length; i++) - addDeleteQuery(queries[i], numDocsInRAM); - return timeToFlushDeletes(); - } + // Immediately mark this document as deleted + // since likely it was partially added. This + // keeps indexing as "all or none" (atomic) when + // adding a document: + addDeleteDocID(state.docID); + } + } + } + } catch (AbortException ae) { + abort(ae); + } - synchronized boolean bufferDeleteQuery(Query query) throws IOException { - waitReady(null); - addDeleteQuery(query, numDocsInRAM); - return timeToFlushDeletes(); - } + return state.doFlushAfter || timeToFlushDeletes(); + } - synchronized boolean deletesFull() { - return maxBufferedDeleteTerms != IndexWriter.DISABLE_AUTO_FLUSH - && ((deletesInRAM.numTerms + deletesInRAM.queries.size() + deletesInRAM.docIDs.size()) >= maxBufferedDeleteTerms); - } + // for testing + synchronized int getNumBufferedDeleteTerms() { + return deletesInRAM.numTerms; + } - synchronized private boolean timeToFlushDeletes() { - return (bufferIsFull || deletesFull()) && setFlushPending(); - } + // for testing + synchronized HashMap getBufferedDeleteTerms() { + return deletesInRAM.terms; + } - void setMaxBufferedDeleteTerms(int maxBufferedDeleteTerms) { - this.maxBufferedDeleteTerms = maxBufferedDeleteTerms; - } + /** Called whenever a merge has completed and the merged segments had deletions */ + synchronized void remapDeletes(SegmentInfos infos, int[][] docMaps, + int[] delCounts, MergePolicy.OneMerge merge, int mergeDocCount) { + if (docMaps == null) + // The merged segments had no deletes so docIDs did not change and we have + // nothing to do + return; + MergeDocIDRemapper mapper = new MergeDocIDRemapper(infos, docMaps, + delCounts, merge, mergeDocCount); + deletesInRAM.remap(mapper, infos, docMaps, delCounts, merge, mergeDocCount); + deletesFlushed.remap(mapper, infos, docMaps, delCounts, merge, + mergeDocCount); + flushedDocCount -= mapper.docShift; + } - int getMaxBufferedDeleteTerms() { - return maxBufferedDeleteTerms; - } + synchronized private void waitReady(DocumentsWriterThreadState state) { + while (!closed + && ((state != null && !state.isIdle) || pauseThreads != 0 + || flushPending || abortCount > 0)) + try { + wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } - synchronized boolean hasDeletes() { - return deletesFlushed.any(); - } + if (closed) + throw new AlreadyClosedException("this IndexWriter is closed"); + } - synchronized boolean applyDeletes(SegmentInfos infos) throws IOException { + synchronized boolean bufferDeleteTerms(Term[] terms) throws IOException { + waitReady(null); + for (int i = 0; i < terms.length; i++) + addDeleteTerm(terms[i], numDocsInRAM); + return timeToFlushDeletes(); + } - if (!hasDeletes()) - return false; + synchronized boolean bufferDeleteTerm(Term term) throws IOException { + waitReady(null); + addDeleteTerm(term, numDocsInRAM); + return timeToFlushDeletes(); + } - if (infoStream != null) - message("apply " + deletesFlushed.numTerms + " buffered deleted terms and " + - deletesFlushed.docIDs.size() + " deleted docIDs and " + - deletesFlushed.queries.size() + " deleted queries on " + - + infos.size() + " segments."); + synchronized boolean bufferDeleteQueries(Query[] queries) throws IOException { + waitReady(null); + for (int i = 0; i < queries.length; i++) + addDeleteQuery(queries[i], numDocsInRAM); + return timeToFlushDeletes(); + } - final int infosEnd = infos.size(); + synchronized boolean bufferDeleteQuery(Query query) throws IOException { + waitReady(null); + addDeleteQuery(query, numDocsInRAM); + return timeToFlushDeletes(); + } - int docStart = 0; - boolean any = false; - for (int i = 0; i < infosEnd; i++) { - IndexReader reader = SegmentReader.get(infos.info(i), false); - boolean success = false; - try { - any |= applyDeletes(reader, docStart); - docStart += reader.maxDoc(); - success = true; - } finally { - if (reader != null) { - try { - if (success) - reader.doCommit(); - } finally { - reader.doClose(); - } - } - } - } + synchronized boolean deletesFull() { + return maxBufferedDeleteTerms != IndexWriter.DISABLE_AUTO_FLUSH + && ((deletesInRAM.numTerms + deletesInRAM.queries.size() + deletesInRAM.docIDs + .size()) >= maxBufferedDeleteTerms); + } - deletesFlushed.clear(); + synchronized private boolean timeToFlushDeletes() { + return (bufferIsFull || deletesFull()) && setFlushPending(); + } - return any; - } + void setMaxBufferedDeleteTerms(int maxBufferedDeleteTerms) { + this.maxBufferedDeleteTerms = maxBufferedDeleteTerms; + } - // Apply buffered delete terms, queries and docIDs to the - // provided reader - private final synchronized boolean applyDeletes(IndexReader reader, int docIDStart) - throws CorruptIndexException, IOException { + int getMaxBufferedDeleteTerms() { + return maxBufferedDeleteTerms; + } - final int docEnd = docIDStart + reader.maxDoc(); - boolean any = false; + synchronized boolean hasDeletes() { + return deletesFlushed.any(); + } - // Delete by term - Iterator iter = deletesFlushed.terms.entrySet().iterator(); - while (iter.hasNext()) { - Entry entry = (Entry) iter.next(); - Term term = (Term) entry.getKey(); + synchronized boolean applyDeletes(SegmentInfos infos) throws IOException { - TermDocs docs = reader.termDocs(term); - if (docs != null) { - int limit = ((BufferedDeletes.Num) entry.getValue()).getNum(); - try { - while (docs.next()) { - int docID = docs.doc(); - if (docIDStart+docID >= limit) - break; - reader.deleteDocument(docID); - any = true; - } - } finally { - docs.close(); - } - } - } + if (!hasDeletes()) + return false; - // Delete by docID - iter = deletesFlushed.docIDs.iterator(); - while(iter.hasNext()) { - int docID = ((Integer) iter.next()).intValue(); - if (docID >= docIDStart && docID < docEnd) { - reader.deleteDocument(docID-docIDStart); - any = true; - } - } + if (infoStream != null) + message("apply " + deletesFlushed.numTerms + + " buffered deleted terms and " + deletesFlushed.docIDs.size() + + " deleted docIDs and " + deletesFlushed.queries.size() + + " deleted queries on " + +infos.size() + " segments."); - // Delete by query - IndexSearcher searcher = new IndexSearcher(reader); - iter = deletesFlushed.queries.entrySet().iterator(); - while(iter.hasNext()) { - Entry entry = (Entry) iter.next(); - Query query = (Query) entry.getKey(); - int limit = ((Integer) entry.getValue()).intValue(); - Weight weight = query.weight(searcher); - Scorer scorer = weight.scorer(reader); - while(scorer.next()) { - final int docID = scorer.doc(); - if (docIDStart + docID >= limit) - break; - reader.deleteDocument(docID); - any = true; - } - } - searcher.close(); - return any; - } + final int infosEnd = infos.size(); - // Buffer a term in bufferedDeleteTerms, which records the - // current number of documents buffered in ram so that the - // delete term will be applied to those documents as well - // as the disk segments. - synchronized private void addDeleteTerm(Term term, int docCount) { - BufferedDeletes.Num num = (BufferedDeletes.Num) deletesInRAM.terms.get(term); - final int docIDUpto = flushedDocCount + docCount; - if (num == null) - deletesInRAM.terms.put(term, new BufferedDeletes.Num(docIDUpto)); - else - num.setNum(docIDUpto); - deletesInRAM.numTerms++; - } + int docStart = 0; + boolean any = false; + for (int i = 0; i < infosEnd; i++) { + IndexReader reader = SegmentReader.get(infos.info(i), false); + boolean success = false; + try { + any |= applyDeletes(reader, docStart); + docStart += reader.maxDoc(); + success = true; + } finally { + if (reader != null) { + try { + if (success) + reader.doCommit(); + } finally { + reader.doClose(); + } + } + } + } - // Buffer a specific docID for deletion. Currently only - // used when we hit a exception when adding a document - synchronized private void addDeleteDocID(int docID) { - deletesInRAM.docIDs.add(new Integer(flushedDocCount+docID)); - } + deletesFlushed.clear(); - synchronized private void addDeleteQuery(Query query, int docID) { - deletesInRAM.queries.put(query, new Integer(flushedDocCount + docID)); - } + return any; + } - /** Does the synchronized work to finish/flush the - * inverted document. */ - private synchronized void finishDocument(DocumentsWriterThreadState state) throws IOException, AbortException { - if (abortCount > 0) { - // Forcefully idle this threadstate -- its state will - // be reset by abort() - state.isIdle = true; - notifyAll(); - return; - } + // Apply buffered delete terms, queries and docIDs to the + // provided reader + private final synchronized boolean applyDeletes(IndexReader reader, + int docIDStart) throws CorruptIndexException, IOException { - if (ramBufferSize != IndexWriter.DISABLE_AUTO_FLUSH - && numBytesUsed >= ramBufferSize) - balanceRAM(); + final int docEnd = docIDStart + reader.maxDoc(); + boolean any = false; - // Now write the indexed document to the real files. - if (nextWriteDocID == state.docID) { - // It's my turn, so write everything now: - nextWriteDocID++; - state.writeDocument(); - state.isIdle = true; - notifyAll(); + // Delete by term + Iterator iter = deletesFlushed.terms.entrySet().iterator(); + while (iter.hasNext()) { + Entry entry = (Entry) iter.next(); + Term term = (Term) entry.getKey(); - // If any states were waiting on me, sweep through and - // flush those that are enabled by my write. - if (numWaiting > 0) { - boolean any = true; - while(any) { - any = false; - for(int i=0;i i+1) - // Swap in the last waiting state to fill in - // the hole we just created. It's important - // to do this as-we-go and not at the end of - // the loop, because if we hit an aborting - // exception in one of the s.writeDocument - // calls (above), it leaves this array in an - // inconsistent state: - waitingThreadStates[i] = waitingThreadStates[numWaiting-1]; - numWaiting--; - } else { - assert !s.isIdle; - i++; - } - } - } - } - } else { - // Another thread got a docID before me, but, it - // hasn't finished its processing. So add myself to - // the line but don't hold up this thread. - waitingThreadStates[numWaiting++] = state; - } - } + TermDocs docs = reader.termDocs(term); + if (docs != null) { + int limit = ((BufferedDeletes.Num) entry.getValue()).getNum(); + try { + while (docs.next()) { + int docID = docs.doc(); + if (docIDStart + docID >= limit) + break; + reader.deleteDocument(docID); + any = true; + } + } finally { + docs.close(); + } + } + } - long getRAMUsed() { - return numBytesUsed; - } + // Delete by docID + iter = deletesFlushed.docIDs.iterator(); + while (iter.hasNext()) { + int docID = ((Integer) iter.next()).intValue(); + if (docID >= docIDStart && docID < docEnd) { + reader.deleteDocument(docID - docIDStart); + any = true; + } + } - long numBytesAlloc; - long numBytesUsed; + // Delete by query + IndexSearcher searcher = new IndexSearcher(reader); + iter = deletesFlushed.queries.entrySet().iterator(); + while (iter.hasNext()) { + Entry entry = (Entry) iter.next(); + Query query = (Query) entry.getKey(); + int limit = ((Integer) entry.getValue()).intValue(); + Weight weight = query.weight(searcher); + Scorer scorer = weight.scorer(reader); + while (scorer.next()) { + final int docID = scorer.doc(); + if (docIDStart + docID >= limit) + break; + reader.deleteDocument(docID); + any = true; + } + } + searcher.close(); + return any; + } - NumberFormat nf = NumberFormat.getInstance(); + // Buffer a term in bufferedDeleteTerms, which records the + // current number of documents buffered in ram so that the + // delete term will be applied to those documents as well + // as the disk segments. + synchronized private void addDeleteTerm(Term term, int docCount) { + BufferedDeletes.Num num = (BufferedDeletes.Num) deletesInRAM.terms + .get(term); + final int docIDUpto = flushedDocCount + docCount; + if (num == null) + deletesInRAM.terms.put(term, new BufferedDeletes.Num(docIDUpto)); + else + num.setNum(docIDUpto); + deletesInRAM.numTerms++; + } - /* Used only when writing norms to fill in default norm - * value into the holes in docID stream for those docs - * that didn't have this field. */ - static void fillBytes(IndexOutput out, byte b, int numBytes) throws IOException { - for(int i=0;i - // IndexOutput - while(numBytes > 0) { - final int chunk; - if (numBytes > 4096) - chunk = 4096; - else - chunk = (int) numBytes; - srcIn.readBytes(copyByteBuffer, 0, chunk); - destIn.writeBytes(copyByteBuffer, 0, chunk); - numBytes -= chunk; - } - } + /** + * Does the synchronized work to finish/flush the inverted document. + */ + private synchronized void finishDocument(DocumentsWriterThreadState state) + throws IOException, AbortException { + if (abortCount > 0) { + // Forcefully idle this threadstate -- its state will + // be reset by abort() + state.isIdle = true; + notifyAll(); + return; + } - // Used only when infoStream != null - private long segmentSize(String segmentName) throws IOException { - assert infoStream != null; - - long size = directory.fileLength(segmentName + ".tii") + - directory.fileLength(segmentName + ".tis") + - directory.fileLength(segmentName + ".frq") + - directory.fileLength(segmentName + ".prx"); + if (ramBufferSize != IndexWriter.DISABLE_AUTO_FLUSH + && numBytesUsed >= ramBufferSize) + balanceRAM(); - final String normFileName = segmentName + ".nrm"; - if (directory.fileExists(normFileName)) - size += directory.fileLength(normFileName); + // Now write the indexed document to the real files. + if (nextWriteDocID == state.docID) { + // It's my turn, so write everything now: + nextWriteDocID++; + state.writeDocument(); + state.isIdle = true; + notifyAll(); - return size; - } + // If any states were waiting on me, sweep through and + // flush those that are enabled by my write. + if (numWaiting > 0) { + boolean any = true; + while (any) { + any = false; + for (int i = 0; i < numWaiting;) { + final DocumentsWriterThreadState s = waitingThreadStates[i]; + if (s.docID == nextWriteDocID) { + s.writeDocument(); + s.isIdle = true; + nextWriteDocID++; + any = true; + if (numWaiting > i + 1) + // Swap in the last waiting state to fill in + // the hole we just created. It's important + // to do this as-we-go and not at the end of + // the loop, because if we hit an aborting + // exception in one of the s.writeDocument + // calls (above), it leaves this array in an + // inconsistent state: + waitingThreadStates[i] = waitingThreadStates[numWaiting - 1]; + numWaiting--; + } else { + assert !s.isIdle; + i++; + } + } + } + } + } else { + // Another thread got a docID before me, but, it + // hasn't finished its processing. So add myself to + // the line but don't hold up this thread. + waitingThreadStates[numWaiting++] = state; + } + } - final private static int POINTER_NUM_BYTE = 4; - final private static int INT_NUM_BYTE = 4; - final private static int CHAR_NUM_BYTE = 2; + long getRAMUsed() { + return numBytesUsed; + } - // Why + 5*POINTER_NUM_BYTE below? - // 1: Posting has "vector" field which is a pointer - // 2: Posting is referenced by postingsFreeList array - // 3,4,5: Posting is referenced by postings hash, which - // targets 25-50% fill factor; approximate this - // as 3X # pointers - final static int POSTING_NUM_BYTE = OBJECT_HEADER_BYTES + 9*INT_NUM_BYTE + 5*POINTER_NUM_BYTE; + long numBytesAlloc; + long numBytesUsed; - // Holds free pool of Posting instances - private Posting[] postingsFreeList; - private int postingsFreeCount; - private int postingsAllocCount; + NumberFormat nf = NumberFormat.getInstance(); - /* Allocate more Postings from shared pool */ - synchronized void getPostings(Posting[] postings) { - numBytesUsed += postings.length * POSTING_NUM_BYTE; - final int numToCopy; - if (postingsFreeCount < postings.length) - numToCopy = postingsFreeCount; - else - numToCopy = postings.length; - final int start = postingsFreeCount-numToCopy; - System.arraycopy(postingsFreeList, start, - postings, 0, numToCopy); - postingsFreeCount -= numToCopy; + /* + * Used only when writing norms to fill in default norm value into the holes + * in docID stream for those docs that didn't have this field. + */ + static void fillBytes(IndexOutput out, byte b, int numBytes) + throws IOException { + for (int i = 0; i < numBytes; i++) + out.writeByte(b); + } - // Directly allocate the remainder if any - if (numToCopy < postings.length) { - final int extra = postings.length - numToCopy; - final int newPostingsAllocCount = postingsAllocCount + extra; - if (newPostingsAllocCount > postingsFreeList.length) - postingsFreeList = new Posting[(int) (1.25 * newPostingsAllocCount)]; + final byte[] copyByteBuffer = new byte[4096]; - balanceRAM(); - for(int i=numToCopy;i + // IndexOutput + while (numBytes > 0) { + final int chunk; + if (numBytes > 4096) + chunk = 4096; + else + chunk = (int) numBytes; + srcIn.readBytes(copyByteBuffer, 0, chunk); + destIn.writeBytes(copyByteBuffer, 0, chunk); + numBytes -= chunk; + } + } - synchronized void recyclePostings(Posting[] postings, int numPostings) { - // Move all Postings from this ThreadState back to our - // free list. We pre-allocated this array while we were - // creating Postings to make sure it's large enough - assert postingsFreeCount + numPostings <= postingsFreeList.length; - System.arraycopy(postings, 0, postingsFreeList, postingsFreeCount, numPostings); - postingsFreeCount += numPostings; - } + // Used only when infoStream != null + private long segmentSize(String segmentName) throws IOException { + assert infoStream != null; - /* Initial chunks size of the shared byte[] blocks used to - store postings data */ - final static int BYTE_BLOCK_SHIFT = 15; - final static int BYTE_BLOCK_SIZE = (int) Math.pow(2.0, BYTE_BLOCK_SHIFT); - final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1; - final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK; + long size = directory.fileLength(segmentName + ".tii") + + directory.fileLength(segmentName + ".tis") + + directory.fileLength(segmentName + ".frq") + + directory.fileLength(segmentName + ".prx"); - private ArrayList freeByteBlocks = new ArrayList(); + final String normFileName = segmentName + ".nrm"; + if (directory.fileExists(normFileName)) + size += directory.fileLength(normFileName); - /* Allocate another byte[] from the shared pool */ - synchronized byte[] getByteBlock(boolean trackAllocations) { - final int size = freeByteBlocks.size(); - final byte[] b; - if (0 == size) { - numBytesAlloc += BYTE_BLOCK_SIZE; - balanceRAM(); - b = new byte[BYTE_BLOCK_SIZE]; - } else - b = (byte[]) freeByteBlocks.remove(size-1); - if (trackAllocations) - numBytesUsed += BYTE_BLOCK_SIZE; - assert numBytesUsed <= numBytesAlloc; - return b; - } + return size; + } - /* Return a byte[] to the pool */ - synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) { - for(int i=start;i postingsFreeList.length) + postingsFreeList = new Posting[(int) (1.25 * newPostingsAllocCount)]; - /* Return a char[] to the pool */ - synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) { - for(int i=0;i freeTrigger) { - if (infoStream != null) - message(" RAM: now balance allocations: usedMB=" + toMB(numBytesUsed) + - " vs trigger=" + toMB(flushTrigger) + - " allocMB=" + toMB(numBytesAlloc) + - " vs trigger=" + toMB(freeTrigger) + - " postingsFree=" + toMB(postingsFreeCount*POSTING_NUM_BYTE) + - " byteBlockFree=" + toMB(freeByteBlocks.size()*BYTE_BLOCK_SIZE) + - " charBlockFree=" + toMB(freeCharBlocks.size()*CHAR_BLOCK_SIZE*CHAR_NUM_BYTE)); + /* Return a byte[] to the pool */ + synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) { + for (int i = start; i < end; i++) + freeByteBlocks.add(blocks[i]); + } - // When we've crossed 100% of our target Postings - // RAM usage, try to free up until we're back down - // to 95% - final long startBytesAlloc = numBytesAlloc; + /* + * Initial chunk size of the shared char[] blocks used to store term text + */ + final static int CHAR_BLOCK_SHIFT = 14; + final static int CHAR_BLOCK_SIZE = (int) Math.pow(2.0, CHAR_BLOCK_SHIFT); + final static int CHAR_BLOCK_MASK = CHAR_BLOCK_SIZE - 1; - final int postingsFreeChunk = (int) (BYTE_BLOCK_SIZE / POSTING_NUM_BYTE); + final static int MAX_TERM_LENGTH = CHAR_BLOCK_SIZE - 1; - int iter = 0; + private ArrayList freeCharBlocks = new ArrayList(); - // We free equally from each pool in 64 KB - // chunks until we are below our threshold - // (freeLevel) + /* Allocate another char[] from the shared pool */ + synchronized char[] getCharBlock() { + final int size = freeCharBlocks.size(); + final char[] c; + if (0 == size) { + numBytesAlloc += CHAR_BLOCK_SIZE * CHAR_NUM_BYTE; + balanceRAM(); + c = new char[CHAR_BLOCK_SIZE]; + } else + c = (char[]) freeCharBlocks.remove(size - 1); + numBytesUsed += CHAR_BLOCK_SIZE * CHAR_NUM_BYTE; + assert numBytesUsed <= numBytesAlloc; + return c; + } - while(numBytesAlloc > freeLevel) { - if (0 == freeByteBlocks.size() && 0 == freeCharBlocks.size() && 0 == postingsFreeCount) { - // Nothing else to free -- must flush now. - bufferIsFull = true; - if (infoStream != null) - message(" nothing to free; now set bufferIsFull"); - break; - } + /* Return a char[] to the pool */ + synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) { + for (int i = 0; i < numBlocks; i++) + freeCharBlocks.add(blocks[i]); + } - if ((0 == iter % 3) && freeByteBlocks.size() > 0) { - freeByteBlocks.remove(freeByteBlocks.size()-1); - numBytesAlloc -= BYTE_BLOCK_SIZE; - } + String toMB(long v) { + return nf.format(v / 1024. / 1024.); + } - if ((1 == iter % 3) && freeCharBlocks.size() > 0) { - freeCharBlocks.remove(freeCharBlocks.size()-1); - numBytesAlloc -= CHAR_BLOCK_SIZE * CHAR_NUM_BYTE; - } + /* + * We have three pools of RAM: Postings, byte blocks (holds freq/prox posting + * data) and char blocks (holds characters in the term). Different docs + * require varying amount of storage from these three classes. For example, + * docs with many unique single-occurrence short terms will use up the + * Postings RAM and hardly any of the other two. Whereas docs with very large + * terms will use alot of char blocks RAM and relatively less of the other + * two. This method just frees allocations from the pools once we are + * over-budget, which balances the pools to match the current docs. + */ + synchronized void balanceRAM() { - if ((2 == iter % 3) && postingsFreeCount > 0) { - final int numToFree; - if (postingsFreeCount >= postingsFreeChunk) - numToFree = postingsFreeChunk; - else - numToFree = postingsFreeCount; - Arrays.fill(postingsFreeList, postingsFreeCount-numToFree, postingsFreeCount, null); - postingsFreeCount -= numToFree; - postingsAllocCount -= numToFree; - numBytesAlloc -= numToFree * POSTING_NUM_BYTE; - } + if (ramBufferSize == IndexWriter.DISABLE_AUTO_FLUSH || bufferIsFull) + return; - iter++; - } - - if (infoStream != null) - message(" after free: freedMB=" + nf.format((startBytesAlloc-numBytesAlloc)/1024./1024.) + " usedMB=" + nf.format(numBytesUsed/1024./1024.) + " allocMB=" + nf.format(numBytesAlloc/1024./1024.)); - - } else { - // If we have not crossed the 100% mark, but have - // crossed the 95% mark of RAM we are actually - // using, go ahead and flush. This prevents - // over-allocating and then freeing, with every - // flush. - if (numBytesUsed > flushTrigger) { - if (infoStream != null) - message(" RAM: now flush @ usedMB=" + nf.format(numBytesUsed/1024./1024.) + - " allocMB=" + nf.format(numBytesAlloc/1024./1024.) + - " triggerMB=" + nf.format(flushTrigger/1024./1024.)); + // We free our allocations if we've allocated 5% over + // our allowed RAM buffer + final long freeTrigger = (long) (1.05 * ramBufferSize); + final long freeLevel = (long) (0.95 * ramBufferSize); - bufferIsFull = true; - } - } - } + // We flush when we've used our target usage + final long flushTrigger = (long) ramBufferSize; + + if (numBytesAlloc > freeTrigger) { + if (infoStream != null) + message(" RAM: now balance allocations: usedMB=" + toMB(numBytesUsed) + + " vs trigger=" + toMB(flushTrigger) + " allocMB=" + + toMB(numBytesAlloc) + " vs trigger=" + toMB(freeTrigger) + + " postingsFree=" + toMB(postingsFreeCount * POSTING_NUM_BYTE) + + " byteBlockFree=" + toMB(freeByteBlocks.size() * BYTE_BLOCK_SIZE) + + " charBlockFree=" + + toMB(freeCharBlocks.size() * CHAR_BLOCK_SIZE * CHAR_NUM_BYTE)); + + // When we've crossed 100% of our target Postings + // RAM usage, try to free up until we're back down + // to 95% + final long startBytesAlloc = numBytesAlloc; + + final int postingsFreeChunk = (int) (BYTE_BLOCK_SIZE / POSTING_NUM_BYTE); + + int iter = 0; + + // We free equally from each pool in 64 KB + // chunks until we are below our threshold + // (freeLevel) + + while (numBytesAlloc > freeLevel) { + if (0 == freeByteBlocks.size() && 0 == freeCharBlocks.size() + && 0 == postingsFreeCount) { + // Nothing else to free -- must flush now. + bufferIsFull = true; + if (infoStream != null) + message(" nothing to free; now set bufferIsFull"); + break; + } + + if ((0 == iter % 3) && freeByteBlocks.size() > 0) { + freeByteBlocks.remove(freeByteBlocks.size() - 1); + numBytesAlloc -= BYTE_BLOCK_SIZE; + } + + if ((1 == iter % 3) && freeCharBlocks.size() > 0) { + freeCharBlocks.remove(freeCharBlocks.size() - 1); + numBytesAlloc -= CHAR_BLOCK_SIZE * CHAR_NUM_BYTE; + } + + if ((2 == iter % 3) && postingsFreeCount > 0) { + final int numToFree; + if (postingsFreeCount >= postingsFreeChunk) + numToFree = postingsFreeChunk; + else + numToFree = postingsFreeCount; + Arrays.fill(postingsFreeList, postingsFreeCount - numToFree, + postingsFreeCount, null); + postingsFreeCount -= numToFree; + postingsAllocCount -= numToFree; + numBytesAlloc -= numToFree * POSTING_NUM_BYTE; + } + + iter++; + } + + if (infoStream != null) + message(" after free: freedMB=" + + nf.format((startBytesAlloc - numBytesAlloc) / 1024. / 1024.) + + " usedMB=" + nf.format(numBytesUsed / 1024. / 1024.) + + " allocMB=" + nf.format(numBytesAlloc / 1024. / 1024.)); + + } else { + // If we have not crossed the 100% mark, but have + // crossed the 95% mark of RAM we are actually + // using, go ahead and flush. This prevents + // over-allocating and then freeing, with every + // flush. + if (numBytesUsed > flushTrigger) { + if (infoStream != null) + message(" RAM: now flush @ usedMB=" + + nf.format(numBytesUsed / 1024. / 1024.) + " allocMB=" + + nf.format(numBytesAlloc / 1024. / 1024.) + " triggerMB=" + + nf.format(flushTrigger / 1024. / 1024.)); + + bufferIsFull = true; + } + } + } } Index: org/apache/lucene/index/DocumentsWriterFieldData.java =================================================================== --- org/apache/lucene/index/DocumentsWriterFieldData.java (revision 652841) +++ org/apache/lucene/index/DocumentsWriterFieldData.java (working copy) @@ -50,6 +50,7 @@ boolean doVectorPositions; boolean doVectorOffsets; boolean postingsCompacted; + boolean doTermDocs; int numPostings; Index: org/apache/lucene/index/DocumentsWriterThreadState.java =================================================================== --- org/apache/lucene/index/DocumentsWriterThreadState.java (revision 652841) +++ org/apache/lucene/index/DocumentsWriterThreadState.java (working copy) @@ -206,7 +206,7 @@ FieldInfo fi = docWriter.fieldInfos.add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), - field.getOmitNorms(), false); + field.getOmitNorms(), false, field.isStoreTermDocs()); if (fi.isIndexed && !fi.omitNorms) { // Maybe grow our buffered norms if (docWriter.norms.length <= fi.number) { @@ -270,6 +270,7 @@ fp.fieldCount = 0; fp.doVectors = fp.doVectorPositions = fp.doVectorOffsets = false; fp.doNorms = fi.isIndexed && !fi.omitNorms; + fp.doTermDocs = fi.storeTermDoc; if (numFieldData == fieldDataArray.length) { int newSize = fieldDataArray.length*2; Index: org/apache/lucene/index/FieldInfo.java =================================================================== --- org/apache/lucene/index/FieldInfo.java (revision 652841) +++ org/apache/lucene/index/FieldInfo.java (working copy) @@ -26,6 +26,7 @@ boolean storeTermVector; boolean storeOffsetWithTermVector; boolean storePositionWithTermVector; + boolean storeTermDoc; boolean omitNorms; // omit norms associated with indexed fields @@ -33,7 +34,7 @@ FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, - boolean omitNorms, boolean storePayloads) { + boolean omitNorms, boolean storePayloads, boolean storeTermDoc) { name = na; isIndexed = tk; number = nu; @@ -42,10 +43,11 @@ this.storePositionWithTermVector = storePositionWithTermVector; this.omitNorms = omitNorms; this.storePayloads = storePayloads; + this.storeTermDoc = storeTermDoc; } public Object clone() { return new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, storePayloads); + storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDoc); } } Index: org/apache/lucene/index/FieldInfos.java =================================================================== --- org/apache/lucene/index/FieldInfos.java (revision 652841) +++ org/apache/lucene/index/FieldInfos.java (working copy) @@ -40,6 +40,7 @@ static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x8; static final byte OMIT_NORMS = 0x10; static final byte STORE_PAYLOADS = 0x20; + static final byte STORE_TERM_DOCS = 0x40; private ArrayList byNumber = new ArrayList(); private HashMap byName = new HashMap(); @@ -83,7 +84,7 @@ while (fieldIterator.hasNext()) { Fieldable field = (Fieldable) fieldIterator.next(); add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), - field.isStoreOffsetWithTermVector(), field.getOmitNorms()); + field.isStoreOffsetWithTermVector(), field.getOmitNorms(), field.isStoreTermDocs()); } } @@ -96,10 +97,10 @@ * @param storeOffsetWithTermVector true if offsets should be stored */ public void addIndexed(Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector) { + boolean storeOffsetWithTermVector, boolean storeTermDocs) { Iterator i = names.iterator(); while (i.hasNext()) { - add((String)i.next(), true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector); + add((String)i.next(), true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, storeTermDocs); } } @@ -126,7 +127,7 @@ * @see #add(String, boolean, boolean, boolean, boolean) */ public void add(String name, boolean isIndexed) { - add(name, isIndexed, false, false, false, false); + add(name, isIndexed, false, false, false, false, false); } /** @@ -137,7 +138,7 @@ * @param storeTermVector true if the term vector should be stored */ public void add(String name, boolean isIndexed, boolean storeTermVector){ - add(name, isIndexed, storeTermVector, false, false, false); + add(name, isIndexed, storeTermVector, false, false, false, false); } /** If the field is not yet known, adds it. If it is known, checks to make @@ -152,9 +153,9 @@ * @param storeOffsetWithTermVector true if the term vector with offsets should be stored */ public void add(String name, boolean isIndexed, boolean storeTermVector, - boolean storePositionWithTermVector, boolean storeOffsetWithTermVector) { + boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean storeTermDocs) { - add(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, false); + add(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, false, storeTermDocs); } /** If the field is not yet known, adds it. If it is known, checks to make @@ -170,9 +171,9 @@ * @param omitNorms true if the norms for the indexed field should be omitted */ public void add(String name, boolean isIndexed, boolean storeTermVector, - boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms) { + boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms, boolean storeTermDocs) { add(name, isIndexed, storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, false); + storeOffsetWithTermVector, omitNorms, false, storeTermDocs); } /** If the field is not yet known, adds it. If it is known, checks to make @@ -190,10 +191,10 @@ */ public FieldInfo add(String name, boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, - boolean omitNorms, boolean storePayloads) { + boolean omitNorms, boolean storePayloads, boolean storeTermDocs) { FieldInfo fi = fieldInfo(name); if (fi == null) { - return addInternal(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads); + return addInternal(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDocs); } else { if (fi.isIndexed != isIndexed) { fi.isIndexed = true; // once indexed, always index @@ -213,17 +214,19 @@ if (fi.storePayloads != storePayloads) { fi.storePayloads = true; } - + if (fi.storeTermDoc != storeTermDocs) { + fi.storeTermDoc = true; + } } return fi; } private FieldInfo addInternal(String name, boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads) { + boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean storeTermDocs) { FieldInfo fi = new FieldInfo(name, isIndexed, byNumber.size(), storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, storePayloads); + storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDocs); byNumber.add(fi); byName.put(name, fi); return fi; @@ -295,6 +298,7 @@ if (fi.storeOffsetWithTermVector) bits |= STORE_OFFSET_WITH_TERMVECTOR; if (fi.omitNorms) bits |= OMIT_NORMS; if (fi.storePayloads) bits |= STORE_PAYLOADS; + if (fi.storeTermDoc) bits |= STORE_TERM_DOCS; output.writeString(fi.name); output.writeByte(bits); } @@ -311,8 +315,8 @@ boolean storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0; boolean omitNorms = (bits & OMIT_NORMS) != 0; boolean storePayloads = (bits & STORE_PAYLOADS) != 0; - - addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads); + boolean storeTermDocs = (bits & STORE_TERM_DOCS) != 0; + addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, storeTermDocs); } } Index: org/apache/lucene/index/FieldsReader.java =================================================================== --- org/apache/lucene/index/FieldsReader.java (revision 652841) +++ org/apache/lucene/index/FieldsReader.java (working copy) @@ -280,6 +280,7 @@ Field.Store store = Field.Store.YES; Field.Index index = getIndexType(fi, tokenize); Field.TermVector termVector = getTermVectorType(fi); + Field.TermDocs termDocs = getTermDocsType(fi); Fieldable f; if (compressed) { @@ -298,7 +299,7 @@ fieldsStream.seek(pointer+length); else fieldsStream.skipChars(length); - f = new LazyField(fi.name, store, index, termVector, length, pointer, binary); + f = new LazyField(fi.name, store, index, termVector, termDocs, length, pointer, binary); f.setOmitNorms(fi.omitNorms); } doc.add(f); @@ -397,7 +398,15 @@ } return termVector; } - + + private Field.TermDocs getTermDocsType(FieldInfo fi) { + if (fi.storeTermDoc) { + return Field.TermDocs.STORE; + } else { + return Field.TermDocs.NO; + } + } + private Field.Index getIndexType(FieldInfo fi, boolean tokenize) { Field.Index index; if (fi.isIndexed && tokenize) @@ -418,15 +427,15 @@ private long pointer; public LazyField(String name, Field.Store store, int toRead, long pointer, boolean isBinary) { - super(name, store, Field.Index.NO, Field.TermVector.NO); + super(name, store, Field.Index.NO, Field.TermVector.NO, Field.TermDocs.NO); this.toRead = toRead; this.pointer = pointer; this.isBinary = isBinary; lazy = true; } - public LazyField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, int toRead, long pointer, boolean isBinary) { - super(name, store, index, termVector); + public LazyField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, Field.TermDocs termDocs, int toRead, long pointer, boolean isBinary) { + super(name, store, index, termVector, termDocs); this.toRead = toRead; this.pointer = pointer; this.isBinary = isBinary; Index: org/apache/lucene/index/FilterIndexReader.java =================================================================== --- org/apache/lucene/index/FilterIndexReader.java (revision 652841) +++ org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -87,6 +87,7 @@ public boolean next() throws IOException { return in.next(); } public Term term() { return in.term(); } public int docFreq() { return in.docFreq(); } + public int[] docs() { return in.docs(); } public void close() throws IOException { in.close(); } } @@ -177,7 +178,17 @@ protected void doSetNorm(int d, String f, byte b) throws CorruptIndexException, IOException { in.setNorm(d, f, b); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return in.terms(loadDocs); + } + public TermEnum terms(Term t, boolean loadDocs) throws IOException { + ensureOpen(); + return in.terms(t, loadDocs); + } + public TermEnum terms() throws IOException { ensureOpen(); return in.terms(); Index: org/apache/lucene/index/IndexReader.java =================================================================== --- org/apache/lucene/index/IndexReader.java (revision 652841) +++ org/apache/lucene/index/IndexReader.java (working copy) @@ -89,6 +89,8 @@ public static final FieldOption TERMVECTOR_WITH_OFFSET = new FieldOption ("TERMVECTOR_WITH_OFFSET"); /** All fields with termvectors with offset values and position values enabled */ public static final FieldOption TERMVECTOR_WITH_POSITION_OFFSET = new FieldOption ("TERMVECTOR_WITH_POSITION_OFFSET"); + + public static final FieldOption TERM_DOCS = new FieldOption ("TERMVECTOR_DOCS"); } private boolean closed; @@ -645,6 +647,10 @@ * @throws IOException if there is a low-level IO error */ public abstract TermEnum terms(Term t) throws IOException; + + public abstract TermEnum terms(Term t, boolean loadDocs) throws IOException; + + public abstract TermEnum terms(boolean loadDocs) throws IOException; /** Returns the number of documents containing the term t. * @throws IOException if there is a low-level IO error Index: org/apache/lucene/index/MultiReader.java =================================================================== --- org/apache/lucene/index/MultiReader.java (revision 652841) +++ org/apache/lucene/index/MultiReader.java (working copy) @@ -295,14 +295,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, null); + return new MultiTermEnum(subReaders, starts, null, false); } public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new MultiTermEnum(subReaders, starts, term); + return new MultiTermEnum(subReaders, starts, term, false); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, null, loadDocs); + } + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, term, loadDocs); + } + public int docFreq(Term t) throws IOException { ensureOpen(); int total = 0; // sum freqs in segments Index: org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- org/apache/lucene/index/MultiSegmentReader.java (revision 652841) +++ org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -17,10 +17,6 @@ * limitations under the License. */ -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.store.Directory; - import java.io.IOException; import java.util.Collection; import java.util.HashMap; @@ -30,619 +26,683 @@ import java.util.Map; import java.util.Set; -/** +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IntArrayList; + +/** * An IndexReader which reads indexes with multiple segments. */ class MultiSegmentReader extends DirectoryIndexReader { - protected SegmentReader[] subReaders; - private int[] starts; // 1st docno for each segment - private Hashtable normsCache = new Hashtable(); - private int maxDoc = 0; - private int numDocs = -1; - private boolean hasDeletions = false; + protected SegmentReader[] subReaders; + private int[] starts; // 1st docno for each segment + private Hashtable normsCache = new Hashtable(); + private int maxDoc = 0; + private int numDocs = -1; + private boolean hasDeletions = false; - /** Construct reading the named set of readers. */ - MultiSegmentReader(Directory directory, SegmentInfos sis, boolean closeDirectory) throws IOException { - super(directory, sis, closeDirectory); - // To reduce the chance of hitting FileNotFound - // (and having to retry), we open segments in - // reverse because IndexWriter merges & deletes - // the newest segments first. + /** Construct reading the named set of readers. */ + MultiSegmentReader(Directory directory, SegmentInfos sis, + boolean closeDirectory) throws IOException { + super(directory, sis, closeDirectory); + // To reduce the chance of hitting FileNotFound + // (and having to retry), we open segments in + // reverse because IndexWriter merges & deletes + // the newest segments first. - SegmentReader[] readers = new SegmentReader[sis.size()]; - for (int i = sis.size()-1; i >= 0; i--) { - try { - readers[i] = SegmentReader.get(sis.info(i)); - } catch (IOException e) { - // Close all readers we had opened: - for(i++;i= 0; i--) { + try { + readers[i] = SegmentReader.get(sis.info(i)); + } catch (IOException e) { + // Close all readers we had opened: + for (i++; i < sis.size(); i++) { + try { + readers[i].close(); + } catch (IOException ignore) { + // keep going - we want to clean up as much as possible + } + } + throw e; + } + } - initialize(readers); - } + initialize(readers); + } - /** This contructor is only used for {@link #reopen()} */ - MultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache) throws IOException { - super(directory, infos, closeDirectory); - - // we put the old SegmentReaders in a map, that allows us - // to lookup a reader using its segment name - Map segmentReaders = new HashMap(); + /** This contructor is only used for {@link #reopen()} */ + MultiSegmentReader(Directory directory, SegmentInfos infos, + boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, + Map oldNormsCache) throws IOException { + super(directory, infos, closeDirectory); - if (oldReaders != null) { - // create a Map SegmentName->SegmentReader - for (int i = 0; i < oldReaders.length; i++) { - segmentReaders.put(oldReaders[i].getSegmentName(), new Integer(i)); - } - } - - SegmentReader[] newReaders = new SegmentReader[infos.size()]; - - // remember which readers are shared between the old and the re-opened - // MultiSegmentReader - we have to incRef those readers - boolean[] readerShared = new boolean[infos.size()]; - - for (int i = infos.size() - 1; i>=0; i--) { - // find SegmentReader for this segment - Integer oldReaderIndex = (Integer) segmentReaders.get(infos.info(i).name); - if (oldReaderIndex == null) { - // this is a new segment, no old SegmentReader can be reused - newReaders[i] = null; - } else { - // there is an old reader for this segment - we'll try to reopen it - newReaders[i] = oldReaders[oldReaderIndex.intValue()]; - } + // we put the old SegmentReaders in a map, that allows us + // to lookup a reader using its segment name + Map segmentReaders = new HashMap(); - boolean success = false; - try { - SegmentReader newReader; - if (newReaders[i] == null || infos.info(i).getUseCompoundFile() != newReaders[i].getSegmentInfo().getUseCompoundFile()) { - // this is a new reader; in case we hit an exception we can close it safely - newReader = SegmentReader.get(infos.info(i)); - } else { - newReader = (SegmentReader) newReaders[i].reopenSegment(infos.info(i)); - } - if (newReader == newReaders[i]) { - // this reader will be shared between the old and the new one, - // so we must incRef it - readerShared[i] = true; - newReader.incRef(); - } else { - readerShared[i] = false; - newReaders[i] = newReader; - } - success = true; - } finally { - if (!success) { - for (i++; i < infos.size(); i++) { - if (newReaders[i] != null) { - try { - if (!readerShared[i]) { - // this is a new subReader that is not used by the old one, - // we can close it - newReaders[i].close(); - } else { - // this subReader is also used by the old reader, so instead - // closing we must decRef it - newReaders[i].decRef(); - } - } catch (IOException ignore) { - // keep going - we want to clean up as much as possible - } - } - } - } - } - } - - // initialize the readers to calculate maxDoc before we try to reuse the old normsCache - initialize(newReaders); - - // try to copy unchanged norms from the old normsCache to the new one - if (oldNormsCache != null) { - Iterator it = oldNormsCache.keySet().iterator(); - while (it.hasNext()) { - String field = (String) it.next(); - if (!hasNorms(field)) { - continue; - } - - byte[] oldBytes = (byte[]) oldNormsCache.get(field); - - byte[] bytes = new byte[maxDoc()]; - - for (int i = 0; i < subReaders.length; i++) { - Integer oldReaderIndex = ((Integer) segmentReaders.get(subReaders[i].getSegmentName())); + if (oldReaders != null) { + // create a Map SegmentName->SegmentReader + for (int i = 0; i < oldReaders.length; i++) { + segmentReaders.put(oldReaders[i].getSegmentName(), new Integer(i)); + } + } - // this SegmentReader was not re-opened, we can copy all of its norms - if (oldReaderIndex != null && - (oldReaders[oldReaderIndex.intValue()] == subReaders[i] - || oldReaders[oldReaderIndex.intValue()].norms.get(field) == subReaders[i].norms.get(field))) { - // we don't have to synchronize here: either this constructor is called from a SegmentReader, - // in which case no old norms cache is present, or it is called from MultiReader.reopen(), - // which is synchronized - System.arraycopy(oldBytes, oldStarts[oldReaderIndex.intValue()], bytes, starts[i], starts[i+1] - starts[i]); - } else { - subReaders[i].norms(field, bytes, starts[i]); - } - } - - normsCache.put(field, bytes); // update cache - } - } - } + SegmentReader[] newReaders = new SegmentReader[infos.size()]; - private void initialize(SegmentReader[] subReaders) { - this.subReaders = subReaders; - starts = new int[subReaders.length + 1]; // build starts array - for (int i = 0; i < subReaders.length; i++) { - starts[i] = maxDoc; - maxDoc += subReaders[i].maxDoc(); // compute maxDocs + // remember which readers are shared between the old and the re-opened + // MultiSegmentReader - we have to incRef those readers + boolean[] readerShared = new boolean[infos.size()]; - if (subReaders[i].hasDeletions()) - hasDeletions = true; - } - starts[subReaders.length] = maxDoc; - } + for (int i = infos.size() - 1; i >= 0; i--) { + // find SegmentReader for this segment + Integer oldReaderIndex = (Integer) segmentReaders.get(infos.info(i).name); + if (oldReaderIndex == null) { + // this is a new segment, no old SegmentReader can be reused + newReaders[i] = null; + } else { + // there is an old reader for this segment - we'll try to reopen it + newReaders[i] = oldReaders[oldReaderIndex.intValue()]; + } - protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException { - if (infos.size() == 1) { - // The index has only one segment now, so we can't refresh the MultiSegmentReader. - // Return a new SegmentReader instead - SegmentReader newReader = SegmentReader.get(infos, infos.info(0), false); - return newReader; - } else { - return new MultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache); - } - } + boolean success = false; + try { + SegmentReader newReader; + if (newReaders[i] == null + || infos.info(i).getUseCompoundFile() != newReaders[i] + .getSegmentInfo().getUseCompoundFile()) { + // this is a new reader; in case we hit an exception we can close it + // safely + newReader = SegmentReader.get(infos.info(i)); + } else { + newReader = (SegmentReader) newReaders[i] + .reopenSegment(infos.info(i)); + } + if (newReader == newReaders[i]) { + // this reader will be shared between the old and the new one, + // so we must incRef it + readerShared[i] = true; + newReader.incRef(); + } else { + readerShared[i] = false; + newReaders[i] = newReader; + } + success = true; + } finally { + if (!success) { + for (i++; i < infos.size(); i++) { + if (newReaders[i] != null) { + try { + if (!readerShared[i]) { + // this is a new subReader that is not used by the old one, + // we can close it + newReaders[i].close(); + } else { + // this subReader is also used by the old reader, so instead + // closing we must decRef it + newReaders[i].decRef(); + } + } catch (IOException ignore) { + // keep going - we want to clean up as much as possible + } + } + } + } + } + } - public TermFreqVector[] getTermFreqVectors(int n) throws IOException { - ensureOpen(); - int i = readerIndex(n); // find segment num - return subReaders[i].getTermFreqVectors(n - starts[i]); // dispatch to segment - } + // initialize the readers to calculate maxDoc before we try to reuse the old + // normsCache + initialize(newReaders); - public TermFreqVector getTermFreqVector(int n, String field) - throws IOException { - ensureOpen(); - int i = readerIndex(n); // find segment num - return subReaders[i].getTermFreqVector(n - starts[i], field); - } + // try to copy unchanged norms from the old normsCache to the new one + if (oldNormsCache != null) { + Iterator it = oldNormsCache.keySet().iterator(); + while (it.hasNext()) { + String field = (String) it.next(); + if (!hasNorms(field)) { + continue; + } + byte[] oldBytes = (byte[]) oldNormsCache.get(field); - public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException { - ensureOpen(); - int i = readerIndex(docNumber); // find segment num - subReaders[i].getTermFreqVector(docNumber - starts[i], field, mapper); - } + byte[] bytes = new byte[maxDoc()]; - public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException { - ensureOpen(); - int i = readerIndex(docNumber); // find segment num - subReaders[i].getTermFreqVector(docNumber - starts[i], mapper); - } + for (int i = 0; i < subReaders.length; i++) { + Integer oldReaderIndex = ((Integer) segmentReaders.get(subReaders[i] + .getSegmentName())); - public boolean isOptimized() { - return false; - } - - public synchronized int numDocs() { - // Don't call ensureOpen() here (it could affect performance) - if (numDocs == -1) { // check cache - int n = 0; // cache miss--recompute - for (int i = 0; i < subReaders.length; i++) - n += subReaders[i].numDocs(); // sum from readers - numDocs = n; - } - return numDocs; - } + // this SegmentReader was not re-opened, we can copy all of its norms + if (oldReaderIndex != null + && (oldReaders[oldReaderIndex.intValue()] == subReaders[i] || oldReaders[oldReaderIndex + .intValue()].norms.get(field) == subReaders[i].norms + .get(field))) { + // we don't have to synchronize here: either this constructor is + // called from a SegmentReader, + // in which case no old norms cache is present, or it is called from + // MultiReader.reopen(), + // which is synchronized + System.arraycopy(oldBytes, oldStarts[oldReaderIndex.intValue()], + bytes, starts[i], starts[i + 1] - starts[i]); + } else { + subReaders[i].norms(field, bytes, starts[i]); + } + } - public int maxDoc() { - // Don't call ensureOpen() here (it could affect performance) - return maxDoc; - } + normsCache.put(field, bytes); // update cache + } + } + } - // inherit javadoc - public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException { - ensureOpen(); - int i = readerIndex(n); // find segment num - return subReaders[i].document(n - starts[i], fieldSelector); // dispatch to segment reader - } + private void initialize(SegmentReader[] subReaders) { + this.subReaders = subReaders; + starts = new int[subReaders.length + 1]; // build starts array + for (int i = 0; i < subReaders.length; i++) { + starts[i] = maxDoc; + maxDoc += subReaders[i].maxDoc(); // compute maxDocs - public boolean isDeleted(int n) { - // Don't call ensureOpen() here (it could affect performance) - int i = readerIndex(n); // find segment num - return subReaders[i].isDeleted(n - starts[i]); // dispatch to segment reader - } + if (subReaders[i].hasDeletions()) + hasDeletions = true; + } + starts[subReaders.length] = maxDoc; + } - public boolean hasDeletions() { - // Don't call ensureOpen() here (it could affect performance) - return hasDeletions; - } + protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos) + throws CorruptIndexException, IOException { + if (infos.size() == 1) { + // The index has only one segment now, so we can't refresh the + // MultiSegmentReader. + // Return a new SegmentReader instead + SegmentReader newReader = SegmentReader.get(infos, infos.info(0), false); + return newReader; + } else { + return new MultiSegmentReader(directory, infos, closeDirectory, + subReaders, starts, normsCache); + } + } - protected void doDelete(int n) throws CorruptIndexException, IOException { - numDocs = -1; // invalidate cache - int i = readerIndex(n); // find segment num - subReaders[i].deleteDocument(n - starts[i]); // dispatch to segment reader - hasDeletions = true; - } + public TermFreqVector[] getTermFreqVectors(int n) throws IOException { + ensureOpen(); + int i = readerIndex(n); // find segment num + return subReaders[i].getTermFreqVectors(n - starts[i]); // dispatch to + // segment + } - protected void doUndeleteAll() throws CorruptIndexException, IOException { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].undeleteAll(); + public TermFreqVector getTermFreqVector(int n, String field) + throws IOException { + ensureOpen(); + int i = readerIndex(n); // find segment num + return subReaders[i].getTermFreqVector(n - starts[i], field); + } - hasDeletions = false; - numDocs = -1; // invalidate cache - } + public void getTermFreqVector(int docNumber, String field, + TermVectorMapper mapper) throws IOException { + ensureOpen(); + int i = readerIndex(docNumber); // find segment num + subReaders[i].getTermFreqVector(docNumber - starts[i], field, mapper); + } - private int readerIndex(int n) { // find reader for doc n: - return readerIndex(n, this.starts, this.subReaders.length); - } - - static int readerIndex(int n, int[] starts, int numSubReaders) { // find reader for doc n: - int lo = 0; // search starts array - int hi = numSubReaders - 1; // for first element less + public void getTermFreqVector(int docNumber, TermVectorMapper mapper) + throws IOException { + ensureOpen(); + int i = readerIndex(docNumber); // find segment num + subReaders[i].getTermFreqVector(docNumber - starts[i], mapper); + } - while (hi >= lo) { - int mid = (lo + hi) >> 1; - int midValue = starts[mid]; - if (n < midValue) - hi = mid - 1; - else if (n > midValue) - lo = mid + 1; - else { // found a match - while (mid+1 < numSubReaders && starts[mid+1] == midValue) { - mid++; // scan to last match - } - return mid; - } - } - return hi; - } + public boolean isOptimized() { + return false; + } - public boolean hasNorms(String field) throws IOException { - ensureOpen(); - for (int i = 0; i < subReaders.length; i++) { - if (subReaders[i].hasNorms(field)) return true; - } - return false; - } + public synchronized int numDocs() { + // Don't call ensureOpen() here (it could affect performance) + if (numDocs == -1) { // check cache + int n = 0; // cache miss--recompute + for (int i = 0; i < subReaders.length; i++) + n += subReaders[i].numDocs(); // sum from readers + numDocs = n; + } + return numDocs; + } - private byte[] ones; - private byte[] fakeNorms() { - if (ones==null) ones=SegmentReader.createFakeNorms(maxDoc()); - return ones; - } + public int maxDoc() { + // Don't call ensureOpen() here (it could affect performance) + return maxDoc; + } - public synchronized byte[] norms(String field) throws IOException { - ensureOpen(); - byte[] bytes = (byte[])normsCache.get(field); - if (bytes != null) - return bytes; // cache hit - if (!hasNorms(field)) - return fakeNorms(); + // inherit javadoc + public Document document(int n, FieldSelector fieldSelector) + throws CorruptIndexException, IOException { + ensureOpen(); + int i = readerIndex(n); // find segment num + return subReaders[i].document(n - starts[i], fieldSelector); // dispatch to + // segment + // reader + } - bytes = new byte[maxDoc()]; - for (int i = 0; i < subReaders.length; i++) - subReaders[i].norms(field, bytes, starts[i]); - normsCache.put(field, bytes); // update cache - return bytes; - } + public boolean isDeleted(int n) { + // Don't call ensureOpen() here (it could affect performance) + int i = readerIndex(n); // find segment num + return subReaders[i].isDeleted(n - starts[i]); // dispatch to segment + // reader + } - public synchronized void norms(String field, byte[] result, int offset) - throws IOException { - ensureOpen(); - byte[] bytes = (byte[])normsCache.get(field); - if (bytes==null && !hasNorms(field)) bytes=fakeNorms(); - if (bytes != null) // cache hit - System.arraycopy(bytes, 0, result, offset, maxDoc()); + public boolean hasDeletions() { + // Don't call ensureOpen() here (it could affect performance) + return hasDeletions; + } - for (int i = 0; i < subReaders.length; i++) // read from segments - subReaders[i].norms(field, result, offset + starts[i]); - } + protected void doDelete(int n) throws CorruptIndexException, IOException { + numDocs = -1; // invalidate cache + int i = readerIndex(n); // find segment num + subReaders[i].deleteDocument(n - starts[i]); // dispatch to segment reader + hasDeletions = true; + } - protected void doSetNorm(int n, String field, byte value) - throws CorruptIndexException, IOException { - normsCache.remove(field); // clear cache - int i = readerIndex(n); // find segment num - subReaders[i].setNorm(n-starts[i], field, value); // dispatch - } + protected void doUndeleteAll() throws CorruptIndexException, IOException { + for (int i = 0; i < subReaders.length; i++) + subReaders[i].undeleteAll(); - public TermEnum terms() throws IOException { - ensureOpen(); - return new MultiTermEnum(subReaders, starts, null); - } + hasDeletions = false; + numDocs = -1; // invalidate cache + } - public TermEnum terms(Term term) throws IOException { - ensureOpen(); - return new MultiTermEnum(subReaders, starts, term); - } + private int readerIndex(int n) { // find reader for doc n: + return readerIndex(n, this.starts, this.subReaders.length); + } - public int docFreq(Term t) throws IOException { - ensureOpen(); - int total = 0; // sum freqs in segments - for (int i = 0; i < subReaders.length; i++) - total += subReaders[i].docFreq(t); - return total; - } + static int readerIndex(int n, int[] starts, int numSubReaders) { // find + // reader + // for doc + // n: + int lo = 0; // search starts array + int hi = numSubReaders - 1; // for first element less - public TermDocs termDocs() throws IOException { - ensureOpen(); - return new MultiTermDocs(subReaders, starts); - } + while (hi >= lo) { + int mid = (lo + hi) >> 1; + int midValue = starts[mid]; + if (n < midValue) + hi = mid - 1; + else if (n > midValue) + lo = mid + 1; + else { // found a match + while (mid + 1 < numSubReaders && starts[mid + 1] == midValue) { + mid++; // scan to last match + } + return mid; + } + } + return hi; + } - public TermPositions termPositions() throws IOException { - ensureOpen(); - return new MultiTermPositions(subReaders, starts); - } + public boolean hasNorms(String field) throws IOException { + ensureOpen(); + for (int i = 0; i < subReaders.length; i++) { + if (subReaders[i].hasNorms(field)) + return true; + } + return false; + } - protected void commitChanges() throws IOException { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].commit(); - } + private byte[] ones; - void startCommit() { - super.startCommit(); - for (int i = 0; i < subReaders.length; i++) { - subReaders[i].startCommit(); - } - } + private byte[] fakeNorms() { + if (ones == null) + ones = SegmentReader.createFakeNorms(maxDoc()); + return ones; + } - void rollbackCommit() { - super.rollbackCommit(); - for (int i = 0; i < subReaders.length; i++) { - subReaders[i].rollbackCommit(); - } - } + public synchronized byte[] norms(String field) throws IOException { + ensureOpen(); + byte[] bytes = (byte[]) normsCache.get(field); + if (bytes != null) + return bytes; // cache hit + if (!hasNorms(field)) + return fakeNorms(); - protected synchronized void doClose() throws IOException { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].decRef(); - - // maybe close directory - super.doClose(); - } + bytes = new byte[maxDoc()]; + for (int i = 0; i < subReaders.length; i++) + subReaders[i].norms(field, bytes, starts[i]); + normsCache.put(field, bytes); // update cache + return bytes; + } - public Collection getFieldNames (IndexReader.FieldOption fieldNames) { - ensureOpen(); - return getFieldNames(fieldNames, this.subReaders); - } - - static Collection getFieldNames (IndexReader.FieldOption fieldNames, IndexReader[] subReaders) { - // maintain a unique set of field names - Set fieldSet = new HashSet(); - for (int i = 0; i < subReaders.length; i++) { - IndexReader reader = subReaders[i]; - Collection names = reader.getFieldNames(fieldNames); - fieldSet.addAll(names); - } - return fieldSet; - } - - // for testing - SegmentReader[] getSubReaders() { - return subReaders; - } + public synchronized void norms(String field, byte[] result, int offset) + throws IOException { + ensureOpen(); + byte[] bytes = (byte[]) normsCache.get(field); + if (bytes == null && !hasNorms(field)) + bytes = fakeNorms(); + if (bytes != null) // cache hit + System.arraycopy(bytes, 0, result, offset, maxDoc()); - public void setTermInfosIndexDivisor(int indexDivisor) throws IllegalStateException { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].setTermInfosIndexDivisor(indexDivisor); - } + for (int i = 0; i < subReaders.length; i++) + // read from segments + subReaders[i].norms(field, result, offset + starts[i]); + } - public int getTermInfosIndexDivisor() throws IllegalStateException { - if (subReaders.length > 0) - return subReaders[0].getTermInfosIndexDivisor(); - else - throw new IllegalStateException("no readers"); - } + protected void doSetNorm(int n, String field, byte value) + throws CorruptIndexException, IOException { + normsCache.remove(field); // clear cache + int i = readerIndex(n); // find segment num + subReaders[i].setNorm(n - starts[i], field, value); // dispatch + } - static class MultiTermEnum extends TermEnum { - private SegmentMergeQueue queue; - - private Term term; - private int docFreq; - - public MultiTermEnum(IndexReader[] readers, int[] starts, Term t) - throws IOException { - queue = new SegmentMergeQueue(readers.length); - for (int i = 0; i < readers.length; i++) { - IndexReader reader = readers[i]; - TermEnum termEnum; - - if (t != null) { - termEnum = reader.terms(t); - } else - termEnum = reader.terms(); - - SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader); - if (t == null ? smi.next() : termEnum.term() != null) - queue.put(smi); // initialize queue - else - smi.close(); - } - - if (t != null && queue.size() > 0) { - next(); - } - } - - public boolean next() throws IOException { - SegmentMergeInfo top = (SegmentMergeInfo)queue.top(); - if (top == null) { - term = null; - return false; - } - - term = top.term; - docFreq = 0; - - while (top != null && term.compareTo(top.term) == 0) { - queue.pop(); - docFreq += top.termEnum.docFreq(); // increment freq - if (top.next()) - queue.put(top); // restore queue - else - top.close(); // done with a segment - top = (SegmentMergeInfo)queue.top(); - } - return true; - } - - public Term term() { - return term; - } - - public int docFreq() { - return docFreq; - } - - public void close() throws IOException { - queue.close(); - } - } + public TermEnum terms() throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, null, false); + } - static class MultiTermDocs implements TermDocs { - protected IndexReader[] readers; - protected int[] starts; - protected Term term; - - protected int base = 0; - protected int pointer = 0; - - private TermDocs[] readerTermDocs; - protected TermDocs current; // == readerTermDocs[pointer] - - public MultiTermDocs(IndexReader[] r, int[] s) { - readers = r; - starts = s; - - readerTermDocs = new TermDocs[r.length]; - } - - public int doc() { - return base + current.doc(); - } - public int freq() { - return current.freq(); - } - - public void seek(Term term) { - this.term = term; - this.base = 0; - this.pointer = 0; - this.current = null; - } - - public void seek(TermEnum termEnum) throws IOException { - seek(termEnum.term()); - } - - public boolean next() throws IOException { - for(;;) { - if (current!=null && current.next()) { - return true; - } - else if (pointer < readers.length) { - base = starts[pointer]; - current = termDocs(pointer++); - } else { - return false; - } - } - } - - /** Optimized implementation. */ - public int read(final int[] docs, final int[] freqs) throws IOException { - while (true) { - while (current == null) { - if (pointer < readers.length) { // try next segment - base = starts[pointer]; - current = termDocs(pointer++); - } else { - return 0; - } - } - int end = current.read(docs, freqs); - if (end == 0) { // none left in segment - current = null; - } else { // got some - final int b = base; // adjust doc numbers - for (int i = 0; i < end; i++) - docs[i] += b; - return end; - } - } - } - - /* A Possible future optimization could skip entire segments */ - public boolean skipTo(int target) throws IOException { - for(;;) { - if (current != null && current.skipTo(target-base)) { - return true; - } else if (pointer < readers.length) { - base = starts[pointer]; - current = termDocs(pointer++); - } else - return false; - } - } - - private TermDocs termDocs(int i) throws IOException { - if (term == null) - return null; - TermDocs result = readerTermDocs[i]; - if (result == null) - result = readerTermDocs[i] = termDocs(readers[i]); - result.seek(term); - return result; - } - - protected TermDocs termDocs(IndexReader reader) - throws IOException { - return reader.termDocs(); - } - - public void close() throws IOException { - for (int i = 0; i < readerTermDocs.length; i++) { - if (readerTermDocs[i] != null) - readerTermDocs[i].close(); - } - } - } + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, null, loadDocs); + } - static class MultiTermPositions extends MultiTermDocs implements TermPositions { - public MultiTermPositions(IndexReader[] r, int[] s) { - super(r,s); - } - - protected TermDocs termDocs(IndexReader reader) throws IOException { - return (TermDocs)reader.termPositions(); - } - - public int nextPosition() throws IOException { - return ((TermPositions)current).nextPosition(); - } - - public int getPayloadLength() { - return ((TermPositions)current).getPayloadLength(); - } - - public byte[] getPayload(byte[] data, int offset) throws IOException { - return ((TermPositions)current).getPayload(data, offset); - } - - - // TODO: Remove warning after API has been finalized - public boolean isPayloadAvailable() { - return ((TermPositions) current).isPayloadAvailable(); - } - } + public TermEnum terms(Term term) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, term, false); + } + + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new MultiTermEnum(subReaders, starts, term, loadDocs); + } + + public int docFreq(Term t) throws IOException { + ensureOpen(); + int total = 0; // sum freqs in segments + for (int i = 0; i < subReaders.length; i++) + total += subReaders[i].docFreq(t); + return total; + } + + public TermDocs termDocs() throws IOException { + ensureOpen(); + return new MultiTermDocs(subReaders, starts); + } + + public TermPositions termPositions() throws IOException { + ensureOpen(); + return new MultiTermPositions(subReaders, starts); + } + + protected void commitChanges() throws IOException { + for (int i = 0; i < subReaders.length; i++) + subReaders[i].commit(); + } + + void startCommit() { + super.startCommit(); + for (int i = 0; i < subReaders.length; i++) { + subReaders[i].startCommit(); + } + } + + void rollbackCommit() { + super.rollbackCommit(); + for (int i = 0; i < subReaders.length; i++) { + subReaders[i].rollbackCommit(); + } + } + + protected synchronized void doClose() throws IOException { + for (int i = 0; i < subReaders.length; i++) + subReaders[i].decRef(); + + // maybe close directory + super.doClose(); + } + + public Collection getFieldNames(IndexReader.FieldOption fieldNames) { + ensureOpen(); + return getFieldNames(fieldNames, this.subReaders); + } + + static Collection getFieldNames(IndexReader.FieldOption fieldNames, + IndexReader[] subReaders) { + // maintain a unique set of field names + Set fieldSet = new HashSet(); + for (int i = 0; i < subReaders.length; i++) { + IndexReader reader = subReaders[i]; + Collection names = reader.getFieldNames(fieldNames); + fieldSet.addAll(names); + } + return fieldSet; + } + + // for testing + SegmentReader[] getSubReaders() { + return subReaders; + } + + public void setTermInfosIndexDivisor(int indexDivisor) + throws IllegalStateException { + for (int i = 0; i < subReaders.length; i++) + subReaders[i].setTermInfosIndexDivisor(indexDivisor); + } + + public int getTermInfosIndexDivisor() throws IllegalStateException { + if (subReaders.length > 0) + return subReaders[0].getTermInfosIndexDivisor(); + else + throw new IllegalStateException("no readers"); + } + + static class MultiTermEnum extends TermEnum { + private SegmentMergeQueue queue; + + private Term term; + private int docFreq; + private int[] docs; + private boolean loadDocs; + private IntArrayList termDocsList = new IntArrayList(50, 1.25); + + public MultiTermEnum(IndexReader[] readers, int[] starts, Term t, + boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; + queue = new SegmentMergeQueue(readers.length); + for (int i = 0; i < readers.length; i++) { + IndexReader reader = readers[i]; + TermEnum termEnum; + + if (t != null) { + termEnum = reader.terms(t, loadDocs); + } else + termEnum = reader.terms(loadDocs); + + SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader); + if (t == null ? smi.next() : termEnum.term() != null) + queue.put(smi); // initialize queue + else + smi.close(); + } + + if (t != null && queue.size() > 0) { + next(); + } + } + + public boolean next() throws IOException { + SegmentMergeInfo top = (SegmentMergeInfo) queue.top(); + if (top == null) { + term = null; + return false; + } + + term = top.term; + docFreq = 0; + termDocsList.clear(); + while (top != null && term.compareTo(top.term) == 0) { + queue.pop(); + docFreq += top.termEnum.docFreq(); // increment freq + // merge doc ids + if (loadDocs) { + int[] readerDocs = top.termEnum.docs(); + if (readerDocs != null) { + for (int x = 0; x < readerDocs.length; x++) { + termDocsList.add(top.base + readerDocs[x]); + } + } + } + if (top.next()) + queue.put(top); // restore queue + else + top.close(); // done with a segment + top = (SegmentMergeInfo) queue.top(); + } + docs = termDocsList.toIntArray(); + return true; + } + + public Term term() { + return term; + } + + public int[] docs() { + return docs; + } + + public int docFreq() { + return docFreq; + } + + public void close() throws IOException { + queue.close(); + } + } + + static class MultiTermDocs implements TermDocs { + protected IndexReader[] readers; + protected int[] starts; + protected Term term; + + protected int base = 0; + protected int pointer = 0; + + private TermDocs[] readerTermDocs; + protected TermDocs current; // == readerTermDocs[pointer] + + public MultiTermDocs(IndexReader[] r, int[] s) { + readers = r; + starts = s; + + readerTermDocs = new TermDocs[r.length]; + } + + public int doc() { + return base + current.doc(); + } + + public int freq() { + return current.freq(); + } + + public void seek(Term term) { + this.term = term; + this.base = 0; + this.pointer = 0; + this.current = null; + } + + public void seek(TermEnum termEnum) throws IOException { + seek(termEnum.term()); + } + + public boolean next() throws IOException { + for (;;) { + if (current != null && current.next()) { + return true; + } else if (pointer < readers.length) { + base = starts[pointer]; + current = termDocs(pointer++); + } else { + return false; + } + } + } + + /** Optimized implementation. */ + public int read(final int[] docs, final int[] freqs) throws IOException { + while (true) { + while (current == null) { + if (pointer < readers.length) { // try next segment + base = starts[pointer]; + current = termDocs(pointer++); + } else { + return 0; + } + } + int end = current.read(docs, freqs); + if (end == 0) { // none left in segment + current = null; + } else { // got some + final int b = base; // adjust doc numbers + for (int i = 0; i < end; i++) + docs[i] += b; + return end; + } + } + } + + /* A Possible future optimization could skip entire segments */ + public boolean skipTo(int target) throws IOException { + for (;;) { + if (current != null && current.skipTo(target - base)) { + return true; + } else if (pointer < readers.length) { + base = starts[pointer]; + current = termDocs(pointer++); + } else + return false; + } + } + + private TermDocs termDocs(int i) throws IOException { + if (term == null) + return null; + TermDocs result = readerTermDocs[i]; + if (result == null) + result = readerTermDocs[i] = termDocs(readers[i]); + result.seek(term); + return result; + } + + protected TermDocs termDocs(IndexReader reader) throws IOException { + return reader.termDocs(); + } + + public void close() throws IOException { + for (int i = 0; i < readerTermDocs.length; i++) { + if (readerTermDocs[i] != null) + readerTermDocs[i].close(); + } + } + } + + static class MultiTermPositions extends MultiTermDocs implements + TermPositions { + public MultiTermPositions(IndexReader[] r, int[] s) { + super(r, s); + } + + protected TermDocs termDocs(IndexReader reader) throws IOException { + return (TermDocs) reader.termPositions(); + } + + public int nextPosition() throws IOException { + return ((TermPositions) current).nextPosition(); + } + + public int getPayloadLength() { + return ((TermPositions) current).getPayloadLength(); + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + return ((TermPositions) current).getPayload(data, offset); + } + + // TODO: Remove warning after API has been finalized + public boolean isPayloadAvailable() { + return ((TermPositions) current).isPayloadAvailable(); + } + } } Index: org/apache/lucene/index/ParallelReader.java =================================================================== --- org/apache/lucene/index/ParallelReader.java (revision 652841) +++ org/apache/lucene/index/ParallelReader.java (working copy) @@ -350,14 +350,24 @@ public TermEnum terms() throws IOException { ensureOpen(); - return new ParallelTermEnum(); + return new ParallelTermEnum(false); } public TermEnum terms(Term term) throws IOException { ensureOpen(); - return new ParallelTermEnum(term); + return new ParallelTermEnum(term, false); } + + public TermEnum terms(boolean loadDocs) throws IOException { + ensureOpen(); + return new ParallelTermEnum(loadDocs); + } + public TermEnum terms(Term term, boolean loadDocs) throws IOException { + ensureOpen(); + return new ParallelTermEnum(term, loadDocs); + } + public int docFreq(Term term) throws IOException { ensureOpen(); IndexReader reader = ((IndexReader)fieldToReader.get(term.field())); @@ -455,18 +465,21 @@ private String field; private Iterator fieldIterator; private TermEnum termEnum; + private boolean loadDocs; - public ParallelTermEnum() throws IOException { + public ParallelTermEnum(boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; field = (String)fieldToReader.firstKey(); if (field != null) - termEnum = ((IndexReader)fieldToReader.get(field)).terms(); + termEnum = ((IndexReader)fieldToReader.get(field)).terms(loadDocs); } - public ParallelTermEnum(Term term) throws IOException { + public ParallelTermEnum(Term term, boolean loadDocs) throws IOException { + this.loadDocs = loadDocs; field = term.field(); IndexReader reader = ((IndexReader)fieldToReader.get(field)); if (reader!=null) - termEnum = reader.terms(term); + termEnum = reader.terms(term, loadDocs); } public boolean next() throws IOException { @@ -510,7 +523,12 @@ return termEnum.docFreq(); } - + + public int[] docs() { + if (termEnum == null) return null; + return termEnum.docs(); + } + public void close() throws IOException { if (termEnum!=null) termEnum.close(); Index: org/apache/lucene/index/SegmentMerger.java =================================================================== --- org/apache/lucene/index/SegmentMerger.java (revision 652841) +++ org/apache/lucene/index/SegmentMerger.java (working copy) @@ -17,16 +17,17 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; import java.util.Vector; -import java.util.Iterator; -import java.util.Collection; -import java.io.IOException; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IntArrayList; /** * The SegmentMerger class combines two or more Segments, represented by an IndexReader ({@link #add}, @@ -64,7 +65,9 @@ /** Maximum number of contiguous documents to bulk-copy when merging stored fields */ private final static int MAX_RAW_MERGE_DOCS = 4192; - + + private IntArrayList termDocsList = new IntArrayList(50, 1.25); + /** This ctor used only by test code. * * @param dir The Directory to merge the other segments into @@ -197,11 +200,11 @@ } private void addIndexed(IndexReader reader, FieldInfos fieldInfos, Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector, boolean storePayloads) throws IOException { + boolean storeOffsetWithTermVector, boolean storePayloads, boolean storeTermDocs) throws IOException { Iterator i = names.iterator(); while (i.hasNext()) { String field = (String)i.next(); - fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads); + fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads, storeTermDocs); } } @@ -264,15 +267,17 @@ SegmentReader segmentReader = (SegmentReader) reader; for (int j = 0; j < segmentReader.getFieldInfos().size(); j++) { FieldInfo fi = segmentReader.getFieldInfos().fieldInfo(j); - fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads); + fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads, fi.storeTermDoc); } } else { - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true); - addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR), true, false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.STORES_PAYLOADS), false, false, false, true, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.INDEXED), false, false, false, false, false); + addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERM_DOCS), false, false, false, false, true); + fieldInfos.add(reader.getFieldNames(IndexReader.FieldOption.UNINDEXED), false); } } @@ -502,7 +507,7 @@ top = (SegmentMergeInfo) queue.top(); } - final int df = mergeTermInfo(match, matchSize); // add new TermInfo + final int df = mergeTermInfo(match, matchSize, term); // add new TermInfo if (checkAbort != null) checkAbort.work(df/3.0); @@ -528,24 +533,39 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int mergeTermInfo(SegmentMergeInfo[] smis, int n) + private final int mergeTermInfo(SegmentMergeInfo[] smis, int n, Term term) throws CorruptIndexException, IOException { long freqPointer = freqOutput.getFilePointer(); long proxPointer = proxOutput.getFilePointer(); + + Holder docsHolder = new Holder(null); + // TODO: can optimize to not generate doc array if no need + int df = appendPostings(smis, n, docsHolder); // append posting data - int df = appendPostings(smis, n); // append posting data - long skipPointer = skipListWriter.writeSkip(freqOutput); if (df > 0) { + int[] docs = null; + FieldInfo fieldInfo = fieldInfos.fieldInfo(term.field()); + if (fieldInfo.storeTermDoc) { + docs = (int[])docsHolder.object; + } // add an entry to the dictionary with pointers to prox and freq files - termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer)); + termInfo.set(df, freqPointer, proxPointer, (int) (skipPointer - freqPointer), docs); termInfosWriter.add(smis[0].term, termInfo); } return df; } + public static class Holder { + public Object object; + + public Holder(Object object) { + this.object = object; + } + } + private byte[] payloadBuffer; private int[][] docMaps; int[][] getDocMaps() { @@ -566,13 +586,14 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int appendPostings(SegmentMergeInfo[] smis, int n) + private final int appendPostings(SegmentMergeInfo[] smis, int n, Holder docsHolder) throws CorruptIndexException, IOException { int lastDoc = 0; int df = 0; // number of docs w/ term skipListWriter.resetSkip(); boolean storePayloads = fieldInfos.fieldInfo(smis[0].term.field).storePayloads; int lastPayloadLength = -1; // ensures that we write the first length + termDocsList.clear(); for (int i = 0; i < n; i++) { SegmentMergeInfo smi = smis[i]; TermPositions postings = smi.getPositions(); @@ -589,7 +610,9 @@ if (doc < 0 || (df > 0 && doc <= lastDoc)) throw new CorruptIndexException("docs out of order (" + doc + " <= " + lastDoc + " )"); - + + termDocsList.add(doc); + df++; if ((df % skipInterval) == 0) { @@ -638,6 +661,7 @@ } } } + docsHolder.object = termDocsList.toIntArray(); return df; } Index: org/apache/lucene/index/SegmentReader.java =================================================================== --- org/apache/lucene/index/SegmentReader.java (revision 652841) +++ org/apache/lucene/index/SegmentReader.java (working copy) @@ -653,7 +653,17 @@ ensureOpen(); return tis.terms(t); } + + public TermEnum terms(boolean loadDocs) { + ensureOpen(); + return tis.terms(loadDocs); + } + public TermEnum terms(Term t, boolean loadDocs) throws IOException { + ensureOpen(); + return tis.terms(t, loadDocs); + } + FieldInfos getFieldInfos() { return fieldInfos; } Index: org/apache/lucene/index/SegmentTermEnum.java =================================================================== --- org/apache/lucene/index/SegmentTermEnum.java (revision 652841) +++ org/apache/lucene/index/SegmentTermEnum.java (working copy) @@ -18,187 +18,228 @@ */ import java.io.IOException; + import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.VIntUtil; final class SegmentTermEnum extends TermEnum implements Cloneable { - private IndexInput input; - FieldInfos fieldInfos; - long size; - long position = -1; + private IndexInput input; + FieldInfos fieldInfos; + long size; + long position = -1; - private TermBuffer termBuffer = new TermBuffer(); - private TermBuffer prevBuffer = new TermBuffer(); - private TermBuffer scanBuffer = new TermBuffer(); // used for scanning + private TermBuffer termBuffer = new TermBuffer(); + private TermBuffer prevBuffer = new TermBuffer(); + private TermBuffer scanBuffer = new TermBuffer(); // used for scanning - private TermInfo termInfo = new TermInfo(); + private TermInfo termInfo = new TermInfo(); - private int format; - private boolean isIndex = false; - long indexPointer = 0; - int indexInterval; - int skipInterval; - int maxSkipLevels; - private int formatM1SkipInterval; + private int format; + private boolean isIndex = false; + long indexPointer = 0; + int indexInterval; + int skipInterval; + int maxSkipLevels; + private int formatM1SkipInterval; + private boolean loadDocs = false; - SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi) - throws CorruptIndexException, IOException { - input = i; - fieldInfos = fis; - isIndex = isi; - maxSkipLevels = 1; // use single-level skip lists for formats > -3 - - int firstInt = input.readInt(); - if (firstInt >= 0) { - // original-format file, without explicit format version number - format = 0; - size = firstInt; + SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi, boolean loadDocs) + throws CorruptIndexException, IOException { + input = i; + fieldInfos = fis; + isIndex = isi; + this.loadDocs = loadDocs; + maxSkipLevels = 1; // use single-level skip lists for formats > -3 - // back-compatible settings - indexInterval = 128; - skipInterval = Integer.MAX_VALUE; // switch off skipTo optimization - } else { - // we have a format version number - format = firstInt; + int firstInt = input.readInt(); + if (firstInt >= 0) { + // original-format file, without explicit format version number + format = 0; + size = firstInt; - // check that it is a format we can understand - if (format < TermInfosWriter.FORMAT_CURRENT) - throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher"); + // back-compatible settings + indexInterval = 128; + skipInterval = Integer.MAX_VALUE; // switch off skipTo optimization + } else { + // we have a format version number + format = firstInt; - size = input.readLong(); // read the size - - if(format == -1){ - if (!isIndex) { - indexInterval = input.readInt(); - formatM1SkipInterval = input.readInt(); - } - // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in - // skipTo implementation of these versions - skipInterval = Integer.MAX_VALUE; - } else { - indexInterval = input.readInt(); - skipInterval = input.readInt(); - if (format <= TermInfosWriter.FORMAT) { - // this new format introduces multi-level skipping - maxSkipLevels = input.readInt(); - } - } - } - if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { - termBuffer.setPreUTF8Strings(); - scanBuffer.setPreUTF8Strings(); - prevBuffer.setPreUTF8Strings(); - } - } + // check that it is a format we can understand + if (format < TermInfosWriter.FORMAT_CURRENT) + throw new CorruptIndexException("Unknown format version:" + format + + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher"); - protected Object clone() { - SegmentTermEnum clone = null; - try { - clone = (SegmentTermEnum) super.clone(); - } catch (CloneNotSupportedException e) {} + size = input.readLong(); // read the size - clone.input = (IndexInput) input.clone(); - clone.termInfo = new TermInfo(termInfo); + if (format == -1) { + if (!isIndex) { + indexInterval = input.readInt(); + formatM1SkipInterval = input.readInt(); + } + // switch off skipTo optimization for file format prior to 1.4rc2 in + // order to avoid a bug in + // skipTo implementation of these versions + skipInterval = Integer.MAX_VALUE; + } else { + indexInterval = input.readInt(); + skipInterval = input.readInt(); + if (format <= TermInfosWriter.FORMAT) { + // this new format introduces multi-level skipping + maxSkipLevels = input.readInt(); + } + } + } + if (format > TermInfosWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) { + termBuffer.setPreUTF8Strings(); + scanBuffer.setPreUTF8Strings(); + prevBuffer.setPreUTF8Strings(); + } + } - clone.termBuffer = (TermBuffer)termBuffer.clone(); - clone.prevBuffer = (TermBuffer)prevBuffer.clone(); - clone.scanBuffer = new TermBuffer(); + void setLoadDocs(boolean loadDocs) { + this.loadDocs = loadDocs; + } - return clone; - } + protected Object clone() { + SegmentTermEnum clone = null; + try { + clone = (SegmentTermEnum) super.clone(); + } catch (CloneNotSupportedException e) { + } - final void seek(long pointer, int p, Term t, TermInfo ti) - throws IOException { - input.seek(pointer); - position = p; - termBuffer.set(t); - prevBuffer.reset(); - termInfo.set(ti); - } + clone.input = (IndexInput) input.clone(); + clone.termInfo = new TermInfo(termInfo); - /** Increments the enumeration to the next element. True if one exists.*/ - public final boolean next() throws IOException { - if (position++ >= size - 1) { - prevBuffer.set(termBuffer); - termBuffer.reset(); - return false; - } + clone.termBuffer = (TermBuffer) termBuffer.clone(); + clone.prevBuffer = (TermBuffer) prevBuffer.clone(); + clone.scanBuffer = new TermBuffer(); + clone.loadDocs = loadDocs; + return clone; + } - prevBuffer.set(termBuffer); - termBuffer.read(input, fieldInfos); + final void seek(long pointer, int p, Term t, TermInfo ti) throws IOException { + input.seek(pointer); + position = p; + termBuffer.set(t); + prevBuffer.reset(); + termInfo.set(ti); + } - termInfo.docFreq = input.readVInt(); // read doc freq - termInfo.freqPointer += input.readVLong(); // read freq pointer - termInfo.proxPointer += input.readVLong(); // read prox pointer + /** Increments the enumeration to the next element. True if one exists. */ + public final boolean next() throws IOException { + if (position++ >= size - 1) { + prevBuffer.set(termBuffer); + termBuffer.reset(); + return false; + } - if(format == -1){ - // just read skipOffset in order to increment file pointer; - // value is never used since skipTo is switched off - if (!isIndex) { - if (termInfo.docFreq > formatM1SkipInterval) { - termInfo.skipOffset = input.readVInt(); - } - } - } - else{ - if (termInfo.docFreq >= skipInterval) - termInfo.skipOffset = input.readVInt(); - } + termInfo.docs = null; + + prevBuffer.set(termBuffer); + termBuffer.read(input, fieldInfos); - if (isIndex) - indexPointer += input.readVLong(); // read index pointer + int bytesLength = input.readVInt(); + if (loadDocs && bytesLength >= 0) { + int numDocs = input.readVInt(); + byte[] bytes = new byte[bytesLength]; + input.readBytes(bytes, 0, bytesLength); + int[] docs = VIntUtil.toArray(numDocs, bytes); + termInfo.docs = docs; + } else { + input.seek(input.getFilePointer() + bytesLength + 1); + } + termInfo.docFreq = input.readVInt(); // read doc freq + termInfo.freqPointer += input.readVLong(); // read freq pointer + termInfo.proxPointer += input.readVLong(); // read prox pointer - return true; - } + if (format == -1) { + // just read skipOffset in order to increment file pointer; + // value is never used since skipTo is switched off + if (!isIndex) { + if (termInfo.docFreq > formatM1SkipInterval) { + termInfo.skipOffset = input.readVInt(); + } + } + } else { + if (termInfo.docFreq >= skipInterval) + termInfo.skipOffset = input.readVInt(); + } - /** Optimized scan, without allocating new terms. */ - final void scanTo(Term term) throws IOException { - scanBuffer.set(term); - while (scanBuffer.compareTo(termBuffer) > 0 && next()) {} - } + if (isIndex) + indexPointer += input.readVLong(); // read index pointer - /** Returns the current Term in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final Term term() { - return termBuffer.toTerm(); - } + return true; + } - /** Returns the previous Term enumerated. Initially null.*/ - final Term prev() { - return prevBuffer.toTerm(); - } + /** Optimized scan, without allocating new terms. */ + final void scanTo(Term term) throws IOException { + scanBuffer.set(term); + while (scanBuffer.compareTo(termBuffer) > 0 && next()) { + } + } - /** Returns the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final TermInfo termInfo() { - return new TermInfo(termInfo); - } + /** + * Returns the current Term in the enumeration. Initially invalid, valid after + * next() called for the first time. + */ + public final Term term() { + return termBuffer.toTerm(); + } - /** Sets the argument to the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final void termInfo(TermInfo ti) { - ti.set(termInfo); - } + /** Returns the previous Term enumerated. Initially null. */ + final Term prev() { + return prevBuffer.toTerm(); + } - /** Returns the docFreq from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final int docFreq() { - return termInfo.docFreq; - } + /** + * Returns the current TermInfo in the enumeration. Initially invalid, valid + * after next() called for the first time. + */ + final TermInfo termInfo() { + return new TermInfo(termInfo); + } - /* Returns the freqPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long freqPointer() { - return termInfo.freqPointer; - } + /** + * Sets the argument to the current TermInfo in the enumeration. Initially + * invalid, valid after next() called for the first time. + */ + final void termInfo(TermInfo ti) { + ti.set(termInfo); + } - /* Returns the proxPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long proxPointer() { - return termInfo.proxPointer; - } + /** + * Returns the docFreq from the current TermInfo in the enumeration. Initially + * invalid, valid after next() called for the first time. + */ + public final int docFreq() { + return termInfo.docFreq; + } - /** Closes the enumeration to further activity, freeing resources. */ - public final void close() throws IOException { - input.close(); - } + /** + * Returns document ids a term is located in + */ + public final int[] docs() { + return termInfo.docs; + } + + /* + * Returns the freqPointer from the current TermInfo in the enumeration. + * Initially invalid, valid after next() called for the first time. + */ + final long freqPointer() { + return termInfo.freqPointer; + } + + /* + * Returns the proxPointer from the current TermInfo in the enumeration. + * Initially invalid, valid after next() called for the first time. + */ + final long proxPointer() { + return termInfo.proxPointer; + } + + /** Closes the enumeration to further activity, freeing resources. */ + public final void close() throws IOException { + input.close(); + } } Index: org/apache/lucene/index/TermEnum.java =================================================================== --- org/apache/lucene/index/TermEnum.java (revision 652841) +++ org/apache/lucene/index/TermEnum.java (working copy) @@ -37,6 +37,8 @@ /** Closes the enumeration to further activity, freeing resources. */ public abstract void close() throws IOException; + public abstract int[] docs(); + // Term Vector support /** Skips terms to the first beyond the current whose value is Index: org/apache/lucene/index/TermInfo.java =================================================================== --- org/apache/lucene/index/TermInfo.java (revision 652841) +++ org/apache/lucene/index/TermInfo.java (working copy) @@ -17,43 +17,54 @@ * limitations under the License. */ -/** A TermInfo is the record of information stored for a term.*/ +/** A TermInfo is the record of information stored for a term. */ final class TermInfo { - /** The number of documents which contain the term. */ - int docFreq = 0; + /** The number of documents which contain the term. */ + int docFreq = 0; - long freqPointer = 0; - long proxPointer = 0; - int skipOffset; + long freqPointer = 0; + long proxPointer = 0; + int skipOffset; + int[] docs; - TermInfo() {} + TermInfo() { + } - TermInfo(int df, long fp, long pp) { - docFreq = df; - freqPointer = fp; - proxPointer = pp; - } + TermInfo(int df, long fp, long pp) { + docFreq = df; + freqPointer = fp; + proxPointer = pp; + } - TermInfo(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } + TermInfo(TermInfo ti) { + docFreq = ti.docFreq; + freqPointer = ti.freqPointer; + proxPointer = ti.proxPointer; + skipOffset = ti.skipOffset; + this.docs = ti.docs; + } + /** + final void set(int docFreq, long freqPointer, long proxPointer, int skipOffset) { + this.docFreq = docFreq; + this.freqPointer = freqPointer; + this.proxPointer = proxPointer; + this.skipOffset = skipOffset; + } + **/ + final void set(int docFreq, long freqPointer, long proxPointer, int skipOffset, int[] docs) { + this.docFreq = docFreq; + this.freqPointer = freqPointer; + this.proxPointer = proxPointer; + this.skipOffset = skipOffset; + this.docs = docs; + } - final void set(int docFreq, - long freqPointer, long proxPointer, int skipOffset) { - this.docFreq = docFreq; - this.freqPointer = freqPointer; - this.proxPointer = proxPointer; - this.skipOffset = skipOffset; - } - - final void set(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } + final void set(TermInfo ti) { + docFreq = ti.docFreq; + freqPointer = ti.freqPointer; + proxPointer = ti.proxPointer; + skipOffset = ti.skipOffset; + docs = ti.docs; + } } Index: org/apache/lucene/index/TermInfosReader.java =================================================================== --- org/apache/lucene/index/TermInfosReader.java (revision 652841) +++ org/apache/lucene/index/TermInfosReader.java (working copy) @@ -22,259 +22,293 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.BufferedIndexInput; -/** This stores a monotonically increasing set of pairs in a - * Directory. Pairs are accessed either by Term or by ordinal position the - * set. */ +/** + * This stores a monotonically increasing set of pairs in a + * Directory. Pairs are accessed either by Term or by ordinal position the set. + */ final class TermInfosReader { - private Directory directory; - private String segment; - private FieldInfos fieldInfos; + private Directory directory; + private String segment; + private FieldInfos fieldInfos; - private ThreadLocal enumerators = new ThreadLocal(); - private SegmentTermEnum origEnum; - private long size; + private ThreadLocal enumerators = new ThreadLocal(); + private SegmentTermEnum origEnum; + private long size; - private Term[] indexTerms = null; - private TermInfo[] indexInfos; - private long[] indexPointers; - - private SegmentTermEnum indexEnum; - - private int indexDivisor = 1; - private int totalIndexInterval; + private Term[] indexTerms = null; + private TermInfo[] indexInfos; + private long[] indexPointers; - TermInfosReader(Directory dir, String seg, FieldInfos fis) - throws CorruptIndexException, IOException { - this(dir, seg, fis, BufferedIndexInput.BUFFER_SIZE); - } + private SegmentTermEnum indexEnum; - TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize) - throws CorruptIndexException, IOException { - boolean success = false; + private int indexDivisor = 1; + private int totalIndexInterval; + private int readBufferSize; - try { - directory = dir; - segment = seg; - fieldInfos = fis; + TermInfosReader(Directory dir, String seg, FieldInfos fis) + throws CorruptIndexException, IOException { + this(dir, seg, fis, BufferedIndexInput.BUFFER_SIZE); + } - origEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_EXTENSION, - readBufferSize), fieldInfos, false); - size = origEnum.size; - totalIndexInterval = origEnum.indexInterval; + TermInfosReader(Directory dir, String seg, FieldInfos fis, + int readBufferSize) throws CorruptIndexException, + IOException { + boolean success = false; + this.readBufferSize = readBufferSize; + try { + directory = dir; + segment = seg; + fieldInfos = fis; - indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, - readBufferSize), fieldInfos, true); + origEnum = new SegmentTermEnum(directory.openInput(segment + "." + + IndexFileNames.TERMS_EXTENSION, readBufferSize), fieldInfos, false, + false); + size = origEnum.size; + totalIndexInterval = origEnum.indexInterval; - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - } + indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + + IndexFileNames.TERMS_INDEX_EXTENSION, readBufferSize), fieldInfos, + true, true); - public int getSkipInterval() { - return origEnum.skipInterval; - } - - public int getMaxSkipLevels() { - return origEnum.maxSkipLevels; - } + success = true; + } finally { + // With lock-less commits, it's entirely possible (and + // fine) to hit a FileNotFound exception above. In + // this case, we want to explicitly close any subset + // of things that were opened so that we don't have to + // wait for a GC to do so. + if (!success) { + close(); + } + } + } - /** - *

    Sets the indexDivisor, which subsamples the number - * of indexed terms loaded into memory. This has a - * similar effect as {@link - * IndexWriter#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1.

    - * - * NOTE: you must call this before the term - * index is loaded. If the index is already loaded, - * an IllegalStateException is thrown. - * - + @throws IllegalStateException if the term index has - * already been loaded into memory. - */ - public void setIndexDivisor(int indexDivisor) throws IllegalStateException { - if (indexDivisor < 1) - throw new IllegalArgumentException("indexDivisor must be > 0: got " + indexDivisor); + public int getSkipInterval() { + return origEnum.skipInterval; + } - if (indexTerms != null) - throw new IllegalStateException("index terms are already loaded"); + public int getMaxSkipLevels() { + return origEnum.maxSkipLevels; + } - this.indexDivisor = indexDivisor; - totalIndexInterval = origEnum.indexInterval * indexDivisor; - } + /** + *

    + * Sets the indexDivisor, which subsamples the number of indexed terms loaded + * into memory. This has a similar effect as {@link + * IndexWriter#setTermIndexInterval} except that setting must be done at + * indexing time while this setting can be set per reader. When set to N, then + * one in every N*termIndexInterval terms in the index is loaded into memory. + * By setting this to a value > 1 you can reduce memory usage, at the expense + * of higher latency when loading a TermInfo. The default value is 1. + *

    + * + * NOTE: you must call this before the term index is loaded. If the + * index is already loaded, an IllegalStateException is thrown. + + * + * @throws IllegalStateException + * if the term index has already been loaded into memory. + */ + public void setIndexDivisor(int indexDivisor) throws IllegalStateException { + if (indexDivisor < 1) + throw new IllegalArgumentException("indexDivisor must be > 0: got " + + indexDivisor); - /** Returns the indexDivisor. - * @see #setIndexDivisor - */ - public int getIndexDivisor() { - return indexDivisor; - } - - final void close() throws IOException { - if (origEnum != null) - origEnum.close(); - if (indexEnum != null) - indexEnum.close(); - enumerators.set(null); - } + if (indexTerms != null) + throw new IllegalStateException("index terms are already loaded"); - /** Returns the number of term/value pairs in the set. */ - final long size() { - return size; - } + this.indexDivisor = indexDivisor; + totalIndexInterval = origEnum.indexInterval * indexDivisor; + } - private SegmentTermEnum getEnum() { - SegmentTermEnum termEnum = (SegmentTermEnum)enumerators.get(); - if (termEnum == null) { - termEnum = terms(); - enumerators.set(termEnum); - } - return termEnum; - } + /** + * Returns the indexDivisor. + * + * @see #setIndexDivisor + */ + public int getIndexDivisor() { + return indexDivisor; + } - private synchronized void ensureIndexIsRead() throws IOException { - if (indexTerms != null) // index already read - return; // do nothing - try { - int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index + final void close() throws IOException { + if (origEnum != null) + origEnum.close(); + if (indexEnum != null) + indexEnum.close(); + enumerators.set(null); + } - indexTerms = new Term[indexSize]; - indexInfos = new TermInfo[indexSize]; - indexPointers = new long[indexSize]; - - for (int i = 0; indexEnum.next(); i++) { - indexTerms[i] = indexEnum.term(); - indexInfos[i] = indexEnum.termInfo(); - indexPointers[i] = indexEnum.indexPointer; - - for (int j = 1; j < indexDivisor; j++) - if (!indexEnum.next()) - break; - } - } finally { - indexEnum.close(); - indexEnum = null; - } - } + /** Returns the number of term/value pairs in the set. */ + final long size() { + return size; + } - /** Returns the offset of the greatest index entry which is less than or equal to term.*/ - private final int getIndexOffset(Term term) { - int lo = 0; // binary search indexTerms[] - int hi = indexTerms.length - 1; + private SegmentTermEnum getEnum() { + SegmentTermEnum termEnum = (SegmentTermEnum) enumerators.get(); + if (termEnum == null) { + termEnum = terms(); + enumerators.set(termEnum); + } + return termEnum; + } - while (hi >= lo) { - int mid = (lo + hi) >> 1; - int delta = term.compareTo(indexTerms[mid]); - if (delta < 0) - hi = mid - 1; - else if (delta > 0) - lo = mid + 1; - else - return mid; - } - return hi; - } + private synchronized void ensureIndexIsRead() throws IOException { + if (indexTerms != null) // index already read + return; // do nothing + try { + int indexSize = 1 + ((int) indexEnum.size - 1) / indexDivisor; // otherwise + // read + // index - private final void seekEnum(int indexOffset) throws IOException { - getEnum().seek(indexPointers[indexOffset], - (indexOffset * totalIndexInterval) - 1, - indexTerms[indexOffset], indexInfos[indexOffset]); - } + indexTerms = new Term[indexSize]; + indexInfos = new TermInfo[indexSize]; + indexPointers = new long[indexSize]; - /** Returns the TermInfo for a Term in the set, or null. */ - TermInfo get(Term term) throws IOException { - if (size == 0) return null; + for (int i = 0; indexEnum.next(); i++) { + indexTerms[i] = indexEnum.term(); + indexInfos[i] = indexEnum.termInfo(); + indexPointers[i] = indexEnum.indexPointer; - ensureIndexIsRead(); + for (int j = 1; j < indexDivisor; j++) + if (!indexEnum.next()) + break; + } + } finally { + indexEnum.close(); + indexEnum = null; + } + } - // optimize sequential access: first try scanning cached enum w/o seeking - SegmentTermEnum enumerator = getEnum(); - if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && term.compareTo(enumerator.prev())> 0) - || term.compareTo(enumerator.term()) >= 0)) { - int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; - if (indexTerms.length == enumOffset // but before end of block - || term.compareTo(indexTerms[enumOffset]) < 0) - return scanEnum(term); // no need to seek - } + /** + * Returns the offset of the greatest index entry which is less than or equal + * to term. + */ + private final int getIndexOffset(Term term) { + int lo = 0; // binary search indexTerms[] + int hi = indexTerms.length - 1; - // random-access: must seek - seekEnum(getIndexOffset(term)); - return scanEnum(term); - } + while (hi >= lo) { + int mid = (lo + hi) >> 1; + int delta = term.compareTo(indexTerms[mid]); + if (delta < 0) + hi = mid - 1; + else if (delta > 0) + lo = mid + 1; + else + return mid; + } + return hi; + } - /** Scans within block for matching term. */ - private final TermInfo scanEnum(Term term) throws IOException { - SegmentTermEnum enumerator = getEnum(); - enumerator.scanTo(term); - if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) - return enumerator.termInfo(); - else - return null; - } + private final void seekEnum(int indexOffset) throws IOException { + getEnum().seek(indexPointers[indexOffset], + (indexOffset * totalIndexInterval) - 1, indexTerms[indexOffset], + indexInfos[indexOffset]); + } + + TermInfo get(Term term) throws IOException { + return get(term, false); + } + + /** Returns the TermInfo for a Term in the set, or null. */ + TermInfo get(Term term, boolean loadDocs) throws IOException { + if (size == 0) + return null; - /** Returns the nth term in the set. */ - final Term get(int position) throws IOException { - if (size == 0) return null; + ensureIndexIsRead(); - SegmentTermEnum enumerator = getEnum(); - if (enumerator != null && enumerator.term() != null && - position >= enumerator.position && - position < (enumerator.position + totalIndexInterval)) - return scanEnum(position); // can avoid seek + // optimize sequential access: first try scanning cached enum w/o seeking + SegmentTermEnum enumerator = getEnum(); + enumerator.setLoadDocs(loadDocs); + if (enumerator.term() != null // term is at or past current + && ((enumerator.prev() != null && term.compareTo(enumerator.prev()) > 0) || term + .compareTo(enumerator.term()) >= 0)) { + int enumOffset = (int) (enumerator.position / totalIndexInterval) + 1; + if (indexTerms.length == enumOffset // but before end of block + || term.compareTo(indexTerms[enumOffset]) < 0) + return scanEnum(term); // no need to seek + } - seekEnum(position/totalIndexInterval); // must seek - return scanEnum(position); - } + // random-access: must seek + seekEnum(getIndexOffset(term)); + return scanEnum(term); + } - private final Term scanEnum(int position) throws IOException { - SegmentTermEnum enumerator = getEnum(); - while(enumerator.position < position) - if (!enumerator.next()) - return null; + /** Scans within block for matching term. */ + private final TermInfo scanEnum(Term term) throws IOException { + SegmentTermEnum enumerator = getEnum(); + enumerator.scanTo(term); + if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) + return enumerator.termInfo(); + else + return null; + } - return enumerator.term(); - } + /** Returns the nth term in the set. */ + final Term get(int position) throws IOException { + if (size == 0) + return null; - /** Returns the position of a Term in the set or -1. */ - final long getPosition(Term term) throws IOException { - if (size == 0) return -1; + SegmentTermEnum enumerator = getEnum(); + if (enumerator != null && enumerator.term() != null + && position >= enumerator.position + && position < (enumerator.position + totalIndexInterval)) + return scanEnum(position); // can avoid seek - ensureIndexIsRead(); - int indexOffset = getIndexOffset(term); - seekEnum(indexOffset); + seekEnum(position / totalIndexInterval); // must seek + return scanEnum(position); + } - SegmentTermEnum enumerator = getEnum(); - while(term.compareTo(enumerator.term()) > 0 && enumerator.next()) {} + private final Term scanEnum(int position) throws IOException { + SegmentTermEnum enumerator = getEnum(); + while (enumerator.position < position) + if (!enumerator.next()) + return null; - if (term.compareTo(enumerator.term()) == 0) - return enumerator.position; - else - return -1; - } + return enumerator.term(); + } - /** Returns an enumeration of all the Terms and TermInfos in the set. */ - public SegmentTermEnum terms() { - return (SegmentTermEnum)origEnum.clone(); - } + /** Returns the position of a Term in the set or -1. */ + final long getPosition(Term term) throws IOException { + if (size == 0) + return -1; - /** Returns an enumeration of terms starting at or after the named term. */ - public SegmentTermEnum terms(Term term) throws IOException { - get(term); - return (SegmentTermEnum)getEnum().clone(); - } + ensureIndexIsRead(); + int indexOffset = getIndexOffset(term); + seekEnum(indexOffset); + + SegmentTermEnum enumerator = getEnum(); + while (term.compareTo(enumerator.term()) > 0 && enumerator.next()) { + } + + if (term.compareTo(enumerator.term()) == 0) + return enumerator.position; + else + return -1; + } + + public SegmentTermEnum terms() { + return (SegmentTermEnum) origEnum.clone(); + } + + /** Returns an enumeration of all the Terms and TermInfos in the set. */ + public SegmentTermEnum terms(boolean loadDocs) { + SegmentTermEnum segmentTermEnum = (SegmentTermEnum) origEnum.clone(); + segmentTermEnum.setLoadDocs(true); + return segmentTermEnum; + } + + /** Returns an enumeration of terms starting at or after the named term. */ + public SegmentTermEnum terms(Term term) throws IOException { + get(term); + return (SegmentTermEnum) getEnum().clone(); + } + + /** Returns an enumeration of terms starting at or after the named term. */ + public SegmentTermEnum terms(Term term, boolean loadDocs) throws IOException { + get(term, loadDocs); + SegmentTermEnum segmentTermEnum = (SegmentTermEnum) getEnum().clone(); + segmentTermEnum.setLoadDocs(loadDocs); + return segmentTermEnum; + } } Index: org/apache/lucene/index/TermInfosWriter.java =================================================================== --- org/apache/lucene/index/TermInfosWriter.java (revision 652841) +++ org/apache/lucene/index/TermInfosWriter.java (working copy) @@ -19,9 +19,12 @@ import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Directory; import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.VIntUtil; /** This stores a monotonically increasing set of pairs in a Directory. A TermInfos can be written once, in order. */ @@ -169,7 +172,15 @@ other.add(lastFieldNumber, lastTermBytes, lastTermBytesLength, lastTi); // add an index term writeTerm(fieldNumber, termBytes, termBytesLength); // write term - + if (ti.docs == null) { + output.writeVInt(-1); + } else { + Arrays.sort(ti.docs); + byte[] bytes = VIntUtil.toBytes(ti.docs); + output.writeVInt(bytes.length); + output.writeVInt(ti.docs.length); + output.writeBytes(bytes, bytes.length); + } output.writeVInt(ti.docFreq); // write doc freq output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers output.writeVLong(ti.proxPointer - lastTi.proxPointer); Index: org/apache/lucene/search/FilteredTermEnum.java =================================================================== --- org/apache/lucene/search/FilteredTermEnum.java (revision 652841) +++ org/apache/lucene/search/FilteredTermEnum.java (working copy) @@ -40,6 +40,10 @@ /** Indicates the end of the enumeration has been reached */ protected abstract boolean endEnum(); + public int[] docs() { + return actualEnum.docs(); + } + protected void setEnum(TermEnum actualEnum) throws IOException { this.actualEnum = actualEnum; // Find the first term that matches Index: org/apache/lucene/util/IntArrayList.java =================================================================== --- org/apache/lucene/util/IntArrayList.java (revision 0) +++ org/apache/lucene/util/IntArrayList.java (revision 0) @@ -0,0 +1,64 @@ +package org.apache.lucene.util; + +public class IntArrayList { + private int[] array; + private int size; + private double growthFactor; + + public IntArrayList(int initialSize, double growthFactor) { + array = new int[initialSize]; + this.growthFactor = growthFactor; + size = 0; + } + + public IntArrayList(int[] array) { + this.array = array; + this.size = array.length; + this.growthFactor = 1.25; + } + + public void setGrowthFactor(double growthFactor) { + this.growthFactor = growthFactor; + } + + public String toString() { + StringBuffer buffer = new StringBuffer(); + for (int x=0; x < size; x++) { + buffer.append(array[x]); + if (x < size-1) + buffer.append(','); + } + return buffer.toString(); + } + + public int size() { + return size; + } + + public void add(int value) { + ensureCapacity(size+1); + array[size] = value; + size++; + } + + public int ensureCapacity(int capacity) { + if (capacity > array.length) { + int newCapacity = (int)(array.length * growthFactor); + int[] newArray = new int[newCapacity]; + System.arraycopy(array, 0, newArray, 0, size); + array = newArray; + capacity = newCapacity; + } + return capacity; + } + + public void clear() { + size = 0; + } + + public int[] toIntArray() { + int[] intArray = new int[size]; + System.arraycopy(array, 0, intArray, 0, size); + return intArray; + } +} Index: org/apache/lucene/util/SortedVIntList.java =================================================================== --- org/apache/lucene/util/SortedVIntList.java (revision 652841) +++ org/apache/lucene/util/SortedVIntList.java (working copy) @@ -39,7 +39,17 @@ private int size; private byte[] bytes; private int lastBytePos; - + + SortedVIntList(int size, byte[] bytes) { + this.size = size; + this.bytes = bytes; + lastBytePos = bytes.length; + } + + byte[] getBytes() { + return bytes; + } + /** * Create a SortedVIntList from all elements of an array of integers. * @@ -105,7 +115,6 @@ builder.done(); } - private class SortedVIntListBuilder { private int lastInt = 0; @@ -166,7 +175,7 @@ public int size() { return size; } - + /** * @return The size of the byte array storing the compressed sorted integers. */ Index: org/apache/lucene/util/VIntUtil.java =================================================================== --- org/apache/lucene/util/VIntUtil.java (revision 0) +++ org/apache/lucene/util/VIntUtil.java (revision 0) @@ -0,0 +1,24 @@ +package org.apache.lucene.util; + +import java.io.IOException; + +import org.apache.lucene.search.DocIdSetIterator; + +public class VIntUtil { + public static byte[] toBytes(int[] array) throws IOException { + SortedVIntList list = new SortedVIntList(array); + return list.getBytes(); + } + + public static int[] toArray(int size, byte[] bytes) throws IOException { + SortedVIntList list = new SortedVIntList(size, bytes); + int[] array = new int[size]; + DocIdSetIterator iterator = list.iterator(); + int x = 0; + while (iterator.next()) { + array[x] = iterator.doc(); + x++; + } + return array; + } +}