Index: lucene/src/java/org/apache/lucene/index/SegmentWriteState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (revision ) @@ -31,9 +31,7 @@ public final Directory directory; public final String segmentName; public final FieldInfos fieldInfos; - public final String docStoreSegmentName; public final int numDocs; - public int numDocsInStore; public boolean hasVectors; public final Collection flushedFiles; @@ -62,15 +60,12 @@ public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, - String docStoreSegmentName, int numDocs, - int numDocsInStore, int termIndexInterval, SegmentCodecs segmentCodecs) { + int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs) { this.infoStream = infoStream; this.directory = directory; this.segmentName = segmentName; this.fieldInfos = fieldInfos; - this.docStoreSegmentName = docStoreSegmentName; this.numDocs = numDocs; - this.numDocsInStore = numDocsInStore; this.termIndexInterval = termIndexInterval; this.segmentCodecs = segmentCodecs; flushedFiles = new HashSet(); @@ -85,9 +80,7 @@ directory = state.directory; segmentName = state.segmentName; fieldInfos = state.fieldInfos; - docStoreSegmentName = state.docStoreSegmentName; numDocs = state.numDocs; - numDocsInStore = state.numDocsInStore; termIndexInterval = state.termIndexInterval; segmentCodecs = state.segmentCodecs; flushedFiles = state.flushedFiles; Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (revision ) @@ -358,47 +358,6 @@ _testMultipleThreadsFailure(new FailOnlyOnAbortOrFlush(true)); } - // Throws IOException during DocumentsWriter.closeDocStore - private static class FailOnlyInCloseDocStore extends MockDirectoryWrapper.Failure { - private boolean onlyOnce; - public FailOnlyInCloseDocStore(boolean onlyOnce) { - this.onlyOnce = onlyOnce; - } - @Override - public void eval(MockDirectoryWrapper dir) throws IOException { - if (doFail) { - StackTraceElement[] trace = new Exception().getStackTrace(); - for (int i = 0; i < trace.length; i++) { - if ("closeDocStore".equals(trace[i].getMethodName())) { - if (onlyOnce) - doFail = false; - throw new IOException("now failing on purpose"); - } - } - } - } - } - - // LUCENE-1130: test IOException in closeDocStore - public void testIOExceptionDuringCloseDocStore() throws IOException { - _testSingleThreadFailure(new FailOnlyInCloseDocStore(false)); - } - - // LUCENE-1130: test IOException in closeDocStore - public void testIOExceptionDuringCloseDocStoreOnlyOnce() throws IOException { - _testSingleThreadFailure(new FailOnlyInCloseDocStore(true)); - } - - // LUCENE-1130: test IOException in closeDocStore, with threads - public void testIOExceptionDuringCloseDocStoreWithThreads() throws Exception { - _testMultipleThreadsFailure(new FailOnlyInCloseDocStore(false)); - } - - // LUCENE-1130: test IOException in closeDocStore, with threads - public void testIOExceptionDuringCloseDocStoreWithThreadsOnlyOnce() throws Exception { - _testMultipleThreadsFailure(new FailOnlyInCloseDocStore(true)); - } - // Throws IOException during DocumentsWriter.writeSegment private static class FailOnlyInWriteSegment extends MockDirectoryWrapper.Failure { private boolean onlyOnce; Index: lucene/src/java/org/apache/lucene/index/MergePolicy.java =================================================================== --- lucene/src/java/org/apache/lucene/index/MergePolicy.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/MergePolicy.java (revision ) @@ -67,7 +67,6 @@ public static class OneMerge { SegmentInfo info; // used by IndexWriter - boolean mergeDocStores; // used by IndexWriter boolean optimize; // used by IndexWriter boolean registerDone; // used by IndexWriter long mergeGen; // used by IndexWriter @@ -153,9 +152,6 @@ b.append(" into ").append(info.name); if (optimize) b.append(" [optimize]"); - if (mergeDocStores) { - b.append(" [mergeDocStores]"); - } if (aborted) { b.append(" [ABORTED]"); } @@ -318,10 +314,4 @@ * Returns true if a new segment (regardless of its origin) should use the compound file format. */ public abstract boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) throws IOException; - - /** - * Returns true if the doc store files should use the - * compound file format. - */ - public abstract boolean useCompoundDocStore(SegmentInfos segments); } Index: lucene/src/java/org/apache/lucene/util/IOUtils.java =================================================================== --- lucene/src/java/org/apache/lucene/util/IOUtils.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/util/IOUtils.java (revision ) @@ -64,4 +64,27 @@ else if (firstIOE != null) throw firstIOE; } + + /** + *

Closes all given Closeables, suppressing all thrown exceptions. Some of the Closeables + * may be null, they are ignored. After everything is closed, method either throws the first of suppressed exceptions, + * or completes normally.

+ * @param objects objects to call close() on + */ + public static void closeSafely(Closeable... objects) throws IOException { + IOException firstIOE = null; + + for (Closeable object : objects) { + try { + if (object != null) + object.close(); + } catch (IOException ioe) { + if (firstIOE == null) + firstIOE = ioe; -} + } + } + + if (firstIOE != null) + throw firstIOE; + } +} Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision ) @@ -157,7 +157,7 @@ String[] startFiles = dir.listAll(); SegmentInfos infos = new SegmentInfos(); infos.read(dir); - new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null, CodecProvider.getDefault()); + new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, CodecProvider.getDefault()); String[] endFiles = dir.listAll(); Arrays.sort(startFiles); @@ -866,7 +866,7 @@ doc.add(newField("field", "aaa", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS)); for(int i=0;i<19;i++) writer.addDocument(doc); - writer.flush(false, true, true); + writer.flush(false, true); writer.close(); SegmentInfos sis = new SegmentInfos(); sis.read(dir); @@ -1039,7 +1039,6 @@ LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setMergeFactor(2); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); Document doc = new Document(); String contents = "aa bb cc dd ee ff gg hh ii jj kk"; @@ -1075,7 +1074,6 @@ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer())); LogMergePolicy lmp2 = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp2.setUseCompoundFile(false); - lmp2.setUseCompoundDocStore(false); writer.optimize(); writer.close(); } @@ -2257,7 +2255,6 @@ TEST_VERSION_CURRENT, new MockAnalyzer())); LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); writer.setInfoStream(new PrintStream(bos)); writer.addDocument(new Document()); Index: lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (revision ) @@ -29,7 +29,6 @@ final DocumentsWriter docWriter; final FieldInfos fieldInfos; int lastDocID; - private String docStoreSegment; PerDoc[] docFreeList = new PerDoc[1]; int freeCount; @@ -44,60 +43,30 @@ } synchronized public void flush(SegmentWriteState state) throws IOException { - - if (state.numDocsInStore > 0) { - // It's possible that all documents seen in this segment - // hit non-aborting exceptions, in which case we will - // not have yet init'd the FieldsWriter: + if (state.numDocs > lastDocID) { initFieldsWriter(); - - // Fill fdx file to include any final docs that we - // skipped because they hit non-aborting exceptions - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); + fill(state.numDocs); } - if (fieldsWriter != null) - fieldsWriter.flush(); - } - - private synchronized void initFieldsWriter() throws IOException { - if (fieldsWriter == null) { - docStoreSegment = docWriter.getDocStoreSegment(); - if (docStoreSegment != null) { - fieldsWriter = new FieldsWriter(docWriter.directory, - docStoreSegment, - fieldInfos); - docWriter.addOpenFile(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_EXTENSION)); - docWriter.addOpenFile(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); - lastDocID = 0; - } - } - } - - synchronized public void closeDocStore(SegmentWriteState state) throws IOException { - final int inc = state.numDocsInStore - lastDocID; - if (inc > 0) { - initFieldsWriter(); - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); - } - if (fieldsWriter != null) { fieldsWriter.close(); fieldsWriter = null; - assert docStoreSegment != null; - assert state.docStoreSegmentName != null; - assert docStoreSegment.equals(state.docStoreSegmentName): "fieldsWriter wrote to segment=" + docStoreSegment + " vs SegmentWriteState segment=" + state.docStoreSegmentName; lastDocID = 0; - String fieldsName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.FIELDS_EXTENSION); - String fieldsIdxName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + + String fieldsName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.FIELDS_EXTENSION); + String fieldsIdxName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.FIELDS_INDEX_EXTENSION); state.flushedFiles.add(fieldsName); state.flushedFiles.add(fieldsIdxName); - docWriter.removeOpenFile(fieldsName); - docWriter.removeOpenFile(fieldsIdxName); + if (4 + ((long) state.numDocs) * 8 != state.directory.fileLength(fieldsIdxName)) + throw new RuntimeException("after flush: fdx size mismatch: " + state.numDocs + " docs vs " + state.directory.fileLength(fieldsIdxName) + " length in bytes of " + fieldsIdxName + " file exists?=" + state.directory.fileExists(fieldsIdxName)); + } + } - if (4+((long) state.numDocsInStore)*8 != state.directory.fileLength(fieldsIdxName)) - throw new RuntimeException("after flush: fdx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.fileLength(fieldsIdxName) + " length in bytes of " + fieldsIdxName + " file exists?=" + state.directory.fileExists(fieldsIdxName)); + private synchronized void initFieldsWriter() throws IOException { + if (fieldsWriter == null) { + fieldsWriter = new FieldsWriter(docWriter.directory, docWriter.getSegment(), fieldInfos); + lastDocID = 0; } } @@ -120,10 +89,7 @@ synchronized void abort() { if (fieldsWriter != null) { - try { - fieldsWriter.close(); - } catch (Throwable t) { - } + fieldsWriter.abort(); fieldsWriter = null; lastDocID = 0; } @@ -131,12 +97,9 @@ /** Fills in any hole in the docIDs */ void fill(int docID) throws IOException { - final int docStoreOffset = docWriter.getDocStoreOffset(); - // We must "catch up" for all docs before us // that had no stored fields: - final int end = docID+docStoreOffset; - while(lastDocID < end) { + while(lastDocID < docID) { fieldsWriter.skipDocument(); lastDocID++; } @@ -156,10 +119,6 @@ assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end"); } - public boolean freeRAM() { - return false; - } - synchronized void free(PerDoc perDoc) { assert freeCount < docFreeList.length; assert 0 == perDoc.numStoredFields; Index: lucene/src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCodecs.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestCodecs.java (revision ) @@ -270,7 +270,7 @@ final Directory dir = newDirectory(); this.write(fieldInfos, dir, fields); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors()); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors()); si.setHasProx(false); final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); @@ -318,7 +318,7 @@ final Directory dir = newDirectory(); this.write(fieldInfos, dir, fields); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors()); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors()); final FieldsProducer terms = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); @@ -607,7 +607,7 @@ final int termIndexInterval = this.nextInt(13, 27); final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()); - final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, codecInfo); + final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, 10000, termIndexInterval, codecInfo); final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state); Arrays.sort(fields); Index: lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (revision ) @@ -137,7 +137,6 @@ iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) .setOpenMode(OpenMode.APPEND).setCodecProvider(provider); ((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundDocStore(false); ((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); @@ -294,4 +293,4 @@ } dir.close(); } -} \ No newline at end of file +} Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision ) @@ -237,7 +237,7 @@ writer.setInfoStream(infoStream); // create the index createIndexNoClose(!optimize, "index1", writer); - writer.flush(false, true, true); + writer.flush(false, true); // create a 2nd index Directory dir2 = newDirectory(); @@ -317,7 +317,7 @@ writer.setInfoStream(infoStream); // create the index createIndexNoClose(!optimize, "index1", writer); - writer.flush(false, true, true); + writer.flush(false, true); // get a reader IndexReader r1 = writer.getReader(); @@ -524,7 +524,7 @@ IndexReader r1 = writer.getReader(); assertEquals(0, r1.maxDoc()); createIndexNoClose(false, "index1", writer); - writer.flush(!optimize, true, true); + writer.flush(!optimize, true); IndexReader iwr1 = writer.getReader(); assertEquals(100, iwr1.maxDoc()); @@ -536,7 +536,7 @@ Document d = createDocument(x, "index1", 5); writer.addDocument(d); } - writer.flush(false, true, true); + writer.flush(false, true); // verify the reader was reopened internally IndexReader iwr2 = writer.getReader(); assertTrue(iwr2 != r1); Index: lucene/src/java/org/apache/lucene/index/TermsHash.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermsHash.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/TermsHash.java (revision ) @@ -70,13 +70,6 @@ } @Override - synchronized void closeDocStore(SegmentWriteState state) throws IOException { - consumer.closeDocStore(state); - if (nextTermsHash != null) - nextTermsHash.closeDocStore(state); - } - - @Override synchronized void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException { Map> childThreadsAndFields = new HashMap>(); Map> nextThreadsAndFields; Index: lucene/src/java/org/apache/lucene/index/NoMergePolicy.java =================================================================== --- lucene/src/java/org/apache/lucene/index/NoMergePolicy.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/NoMergePolicy.java (revision ) @@ -67,9 +67,6 @@ throws CorruptIndexException, IOException { return null; } @Override - public boolean useCompoundDocStore(SegmentInfos segments) { return useCompoundFile; } - - @Override public boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) { return useCompoundFile; } @Override Index: lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (revision ) @@ -20,6 +20,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; import java.io.IOException; @@ -30,7 +31,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { final DocumentsWriter docWriter; - TermVectorsWriter termVectorsWriter; PerDoc[] docFreeList = new PerDoc[1]; int freeCount; IndexOutput tvx; @@ -50,26 +50,28 @@ @Override synchronized void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException { - - // NOTE: it's possible that all documents seen in this segment - // hit non-aborting exceptions, in which case we will - // not have yet init'd the TermVectorsWriter. This is - // actually OK (unlike in the stored fields case) - // because, although FieldInfos.hasVectors() will return - // true, the TermVectorsReader gracefully handles - // non-existence of the term vectors files. - state.hasVectors = hasVectors; - if (tvx != null) { + // At least one doc in this run had term vectors enabled + fill(state.numDocs); + tvx.close(); + tvf.close(); + tvd.close(); + tvx = tvd = tvf = null; + assert state.segmentName != null; + String idxName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.VECTORS_INDEX_EXTENSION); + String fldName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); + String docName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); - if (state.numDocsInStore > 0) - // In case there are some final documents that we - // didn't see (because they hit a non-aborting exception): - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); + if (4 + ((long) state.numDocs) * 16 != state.directory.fileLength(idxName)) + throw new RuntimeException("after flush: tvx size mismatch: " + state.numDocs + " docs vs " + state.directory.fileLength(idxName) + " length in bytes of " + idxName + " file exists?=" + state.directory.fileExists(idxName)); - tvx.flush(); - tvd.flush(); - tvf.flush(); + state.flushedFiles.add(idxName); + state.flushedFiles.add(fldName); + state.flushedFiles.add(docName); + + lastDocID = 0; + state.hasVectors = hasVectors; + hasVectors = false; } for (Map.Entry> entry : threadsAndFields.entrySet()) { @@ -84,37 +86,6 @@ } } - @Override - synchronized void closeDocStore(final SegmentWriteState state) throws IOException { - if (tvx != null) { - // At least one doc in this run had term vectors - // enabled - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); - tvx.close(); - tvf.close(); - tvd.close(); - tvx = null; - assert state.docStoreSegmentName != null; - String idxName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.VECTORS_INDEX_EXTENSION); - if (4+((long) state.numDocsInStore)*16 != state.directory.fileLength(idxName)) - throw new RuntimeException("after flush: tvx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.fileLength(idxName) + " length in bytes of " + idxName + " file exists?=" + state.directory.fileExists(idxName)); - - String fldName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); - String docName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); - state.flushedFiles.add(idxName); - state.flushedFiles.add(fldName); - state.flushedFiles.add(docName); - - docWriter.removeOpenFile(idxName); - docWriter.removeOpenFile(fldName); - docWriter.removeOpenFile(docName); - - lastDocID = 0; - state.hasVectors = hasVectors; - hasVectors = false; - } - } - int allocCount; synchronized PerDoc getPerDoc() { @@ -135,11 +106,9 @@ /** Fills in no-term-vectors for all docs we haven't seen * since the last doc that had term vectors. */ void fill(int docID) throws IOException { - final int docStoreOffset = docWriter.getDocStoreOffset(); - final int end = docID+docStoreOffset; - if (lastDocID < end) { + if (lastDocID < docID) { final long tvfPosition = tvf.getFilePointer(); - while(lastDocID < end) { + while(lastDocID < docID) { tvx.writeLong(tvd.getFilePointer()); tvd.writeVInt(0); tvx.writeLong(tvfPosition); @@ -151,31 +120,19 @@ synchronized void initTermVectorsWriter() throws IOException { if (tvx == null) { - final String docStoreSegment = docWriter.getDocStoreSegment(); - - if (docStoreSegment == null) - return; - // If we hit an exception while init'ing the term // vector output files, we must abort this segment // because those files will be in an unknown // state: - String idxName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_INDEX_EXTENSION); - String docName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); - String fldName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); hasVectors = true; - tvx = docWriter.directory.createOutput(idxName); - tvd = docWriter.directory.createOutput(docName); - tvf = docWriter.directory.createOutput(fldName); + tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION)); + tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)); + tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION)); tvx.writeInt(TermVectorsReader.FORMAT_CURRENT); tvd.writeInt(TermVectorsReader.FORMAT_CURRENT); tvf.writeInt(TermVectorsReader.FORMAT_CURRENT); - docWriter.addOpenFile(idxName); - docWriter.addOpenFile(fldName); - docWriter.addOpenFile(docName); - lastDocID = 0; } } @@ -206,7 +163,7 @@ perDoc.numVectorFields = 0; } - assert lastDocID == perDoc.docID + docWriter.getDocStoreOffset(); + assert lastDocID == perDoc.docID; lastDocID++; @@ -215,36 +172,14 @@ assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end"); } - public boolean freeRAM() { - // We don't hold any state beyond one doc, so we don't - // free persistent RAM here - return false; - } - @Override public void abort() { hasVectors = false; - if (tvx != null) { - try { - tvx.close(); - } catch (Throwable t) { - } - tvx = null; - } - if (tvd != null) { - try { - tvd.close(); - } catch (Throwable t) { - } - tvd = null; - } - if (tvf != null) { - try { - tvf.close(); - } catch (Throwable t) { - } - tvf = null; - } + try { IOUtils.closeSafely(tvx, tvd, tvf); } catch (IOException ignored) {} + try { docWriter.directory.deleteFile(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION)); } catch (IOException ignored) {} + try { docWriter.directory.deleteFile(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)); } catch (IOException ignored) {} + try { docWriter.directory.deleteFile(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION)); } catch (IOException ignored) {} + tvx = tvd = tvf = null; lastDocID = 0; } Index: lucene/src/test/org/apache/lucene/index/TestNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestNorms.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestNorms.java (revision ) @@ -154,7 +154,6 @@ LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(true); - lmp.setUseCompoundDocStore(true); iw.close(); } @@ -199,7 +198,6 @@ LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(compound); - lmp.setUseCompoundDocStore(compound); for (int i = 0; i < ndocs; i++) { iw.addDocument(newDoc()); } Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision ) @@ -496,7 +496,6 @@ Directory dir = newFSDirectory(new File(dirName)); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10); ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS); - ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<35;i++) { @@ -508,7 +507,6 @@ // open fresh writer so we get no prx file in the added segment conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10); ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS); - ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS); writer = new IndexWriter(dir, conf); addNoProxDoc(writer); writer.close(); Index: lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (revision ) @@ -138,7 +138,6 @@ cfg.setCodecProvider(new AppendingCodecProvider()); ((LogMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy)cfg.getMergePolicy()).setUseCompoundDocStore(false); IndexWriter writer = new IndexWriter(dir, cfg); Document doc = new Document(); doc.add(newField("f", text, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS)); Index: lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (revision ) @@ -195,16 +195,13 @@ final double SECONDS = 2.0; - boolean useCompoundFile = true; - Directory dir = newDirectory(); ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) .setIndexDeletionPolicy(policy); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); - lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); + lmp.setUseCompoundFile(true); IndexWriter writer = new IndexWriter(dir, conf); writer.close(); @@ -219,8 +216,7 @@ new MockAnalyzer()).setOpenMode( OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); - lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); + lmp.setUseCompoundFile(true); writer = new IndexWriter(dir, conf); for(int j=0;j<17;j++) { addDoc(writer); @@ -298,7 +294,6 @@ .setMergeScheduler(new SerialMergeScheduler()); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); lmp.setMergeFactor(10); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<107;i++) { @@ -311,7 +306,6 @@ OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); writer.optimize(); writer.close(); @@ -488,7 +482,6 @@ .setMaxBufferedDocs(10); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<107;i++) { addDoc(writer); @@ -499,7 +492,6 @@ .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); writer.optimize(); writer.close(); @@ -539,7 +531,6 @@ .setMaxBufferedDocs(10); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<17;i++) { addDoc(writer); @@ -597,7 +588,6 @@ .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); writer.close(); Term searchTerm = new Term("content", "aaa"); @@ -609,7 +599,6 @@ .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); for(int j=0;j<17;j++) { addDoc(writer); @@ -630,7 +619,6 @@ .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); writer.optimize(); // this is a commit @@ -706,7 +694,6 @@ .setMaxBufferedDocs(10); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); writer.close(); Term searchTerm = new Term("content", "aaa"); @@ -720,7 +707,6 @@ .setMaxBufferedDocs(10); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); for(int j=0;j<17;j++) { addDoc(writer); Index: lucene/src/java/org/apache/lucene/index/DocConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocConsumer.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/DocConsumer.java (revision ) @@ -23,7 +23,6 @@ abstract class DocConsumer { abstract DocConsumerPerThread addThread(DocumentsWriterThreadState perThread) throws IOException; abstract void flush(final Collection threads, final SegmentWriteState state) throws IOException; - abstract void closeDocStore(final SegmentWriteState state) throws IOException; abstract void abort(); abstract boolean freeRAM(); } Index: lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java (revision ) @@ -32,9 +32,6 @@ /** Flush a new segment */ abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; - /** Close doc stores */ - abstract void closeDocStore(SegmentWriteState state) throws IOException; - /** Attempt to free RAM, returning true if any RAM was * freed */ abstract boolean freeRAM(); Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (revision ) @@ -114,12 +114,9 @@ Directory directory; String segment; // Current segment we are working on - private String docStoreSegment; // Current doc-store segment we are writing - private int docStoreOffset; // Current starting doc-store offset of current segment - private int nextDocID; // Next docID to be added + private int nextDocID; // Next docID to be added - private int numDocsInRAM; // # docs buffered in RAM - int numDocsInStore; // # docs written to doc stores + private int numDocs; // # of docs added, but not yet flushed // Max # ThreadState instances; if there are more threads // than this they share ThreadStates @@ -139,8 +136,6 @@ // this, they wait for others to finish first private final int maxThreadStates; - List newFiles; - // Deletes for our still-in-RAM (to be flushed next) segment private SegmentDeletes pendingDeletes = new SegmentDeletes(); @@ -323,7 +318,7 @@ final boolean doFlush = flushControl.waitUpdate(0, queries.length); synchronized(this) { for (Query query : queries) { - pendingDeletes.addQuery(query, numDocsInRAM); + pendingDeletes.addQuery(query, numDocs); } } return doFlush; @@ -332,7 +327,7 @@ boolean deleteQuery(Query query) { final boolean doFlush = flushControl.waitUpdate(0, 1); synchronized(this) { - pendingDeletes.addQuery(query, numDocsInRAM); + pendingDeletes.addQuery(query, numDocs); } return doFlush; } @@ -341,7 +336,7 @@ final boolean doFlush = flushControl.waitUpdate(0, terms.length); synchronized(this) { for (Term term : terms) { - pendingDeletes.addTerm(term, numDocsInRAM); + pendingDeletes.addTerm(term, numDocs); } } return doFlush; @@ -350,7 +345,7 @@ boolean deleteTerm(Term term, boolean skipWait) { final boolean doFlush = flushControl.waitUpdate(0, 1, skipWait); synchronized(this) { - pendingDeletes.addTerm(term, numDocsInRAM); + pendingDeletes.addTerm(term, numDocs); } return doFlush; } @@ -424,128 +419,15 @@ } /** Returns how many docs are currently buffered in RAM. */ - synchronized int getNumDocsInRAM() { - return numDocsInRAM; + synchronized int getNumDocs() { + return numDocs; } - /** Returns the current doc store segment we are writing - * to. */ - synchronized String getDocStoreSegment() { - return docStoreSegment; - } - - /** Returns the doc offset into the shared doc store for - * the current buffered docs. */ - synchronized int getDocStoreOffset() { - return docStoreOffset; - } - - /** Closes the current open doc stores an sets the - * docStoreSegment and docStoreUseCFS on the provided - * SegmentInfo. */ - synchronized void closeDocStore(SegmentWriteState flushState, IndexWriter writer, IndexFileDeleter deleter, SegmentInfo newSegment, MergePolicy mergePolicy, SegmentInfos segmentInfos) throws IOException { - - final boolean isSeparate = numDocsInRAM == 0 || !segment.equals(docStoreSegment); - - assert docStoreSegment != null; - - if (infoStream != null) { - message("closeDocStore: openFiles=" + openFiles + "; segment=" + docStoreSegment + "; docStoreOffset=" + docStoreOffset + "; numDocsInStore=" + numDocsInStore + "; isSeparate=" + isSeparate); - } - - closedFiles.clear(); - consumer.closeDocStore(flushState); - flushState.numDocsInStore = 0; - assert 0 == openFiles.size(); - - if (isSeparate) { - flushState.flushedFiles.clear(); - - if (mergePolicy.useCompoundDocStore(segmentInfos)) { - - final String compoundFileName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION); - - if (infoStream != null) { - message("closeDocStore: create compound file " + compoundFileName); - } - - boolean success = false; - try { - - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); - for (final String file : closedFiles) { - cfsWriter.addFile(file); - } - - // Perform the merge - cfsWriter.close(); - - success = true; - } finally { - if (!success) { - deleter.deleteFile(compoundFileName); - } - } - - // In case the files we just merged into a CFS were - // not registered w/ IFD: - deleter.deleteNewFiles(closedFiles); - - final int numSegments = segmentInfos.size(); - for(int i=0;i openFiles = new ArrayList(); - final List closedFiles = new ArrayList(); - - /* Returns Collection of files in use by this instance, - * including any flushed segments. */ - @SuppressWarnings("unchecked") - synchronized List openFiles() { - return (List) ((ArrayList) openFiles).clone(); - } - - synchronized void addOpenFile(String name) { - assert !openFiles.contains(name); - openFiles.add(name); - } - - synchronized void removeOpenFile(String name) { - assert openFiles.contains(name); - openFiles.remove(name); - closedFiles.add(name); - } - synchronized void setAborting() { if (infoStream != null) { message("setAborting"); @@ -558,7 +440,6 @@ * currently buffered docs. This resets our state, * discarding any docs added since last flush. */ synchronized void abort() throws IOException { - if (infoStream != null) { message("docWriter: abort"); } @@ -582,19 +463,11 @@ waitQueue.waitingBytes = 0; - try { - abortedFiles = openFiles(); - } catch (Throwable t) { - abortedFiles = null; - } - pendingDeletes.clear(); - + - openFiles.clear(); - - for(int i=0;i DW -> BD pushDeletes(newSegment, segmentInfos); - docStoreOffset = numDocsInStore; - return newSegment; } @@ -832,17 +658,6 @@ notifyAll(); } - synchronized void initSegmentName(boolean onlyDocStore) { - if (segment == null && (!onlyDocStore || docStoreSegment == null)) { - segment = writer.newSegmentName(); - assert numDocsInRAM == 0; - } - if (docStoreSegment == null) { - docStoreSegment = segment; - assert numDocsInStore == 0; - } - } - /** Returns a free (idle) ThreadState that may be used for * indexing this one document. This call also pauses if a * flush is pending. If delTerm is non-null then we @@ -888,7 +703,10 @@ // Allocate segment name if this is the first doc since // last flush: - initSegmentName(false); + if (segment == null) { + segment = writer.newSegmentName(); + assert numDocs == 0; + } state.docState.docID = nextDocID++; @@ -896,7 +714,7 @@ pendingDeletes.addTerm(delTerm, state.docState.docID); } - numDocsInRAM++; + numDocs++; state.isIdle = false; return state; } @@ -1267,7 +1085,6 @@ try { doc.finish(); nextWriteDocID++; - numDocsInStore++; nextWriteLoc++; assert nextWriteLoc <= waiting.length; if (nextWriteLoc == waiting.length) Index: lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java =================================================================== --- lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (revision ) @@ -82,7 +82,6 @@ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFiles); - lmp.setUseCompoundDocStore(useCompoundFiles); IndexWriter writer = new IndexWriter(directory, conf); if (VERBOSE) { System.out.println("TEST: now build index"); Index: lucene/src/test/org/apache/lucene/TestSearch.java =================================================================== --- lucene/src/test/org/apache/lucene/TestSearch.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/TestSearch.java (revision ) @@ -77,7 +77,6 @@ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(directory, conf); String[] docs = { Index: lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java (revision ) @@ -24,7 +24,6 @@ abstract class InvertedDocEndConsumer { abstract InvertedDocEndConsumerPerThread addThread(DocInverterPerThread docInverterPerThread); abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; - abstract void closeDocStore(SegmentWriteState state) throws IOException; abstract void abort(); abstract void setFieldInfos(FieldInfos fieldInfos); } Index: lucene/src/test/org/apache/lucene/index/TestDoc.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDoc.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestDoc.java (revision ) @@ -201,7 +201,7 @@ r2.close(); final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir, - useCompoundFile, -1, null, false, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(), + useCompoundFile, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()); if (useCompoundFile) { Index: lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java (revision ) @@ -29,10 +29,6 @@ * segment */ abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; - /** Called when DocumentsWriter decides to close the doc - * stores */ - abstract void closeDocStore(SegmentWriteState state) throws IOException; - /** Called when an aborting exception is hit */ abstract void abort(); Index: lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java (revision ) @@ -25,7 +25,6 @@ abstract TermsHashConsumerPerThread addThread(TermsHashPerThread perThread); abstract void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException; abstract void abort(); - abstract void closeDocStore(SegmentWriteState state) throws IOException; FieldInfos fieldInfos; Index: lucene/src/java/org/apache/lucene/index/DocInverter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocInverter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/DocInverter.java (revision ) @@ -74,12 +74,6 @@ } @Override - public void closeDocStore(SegmentWriteState state) throws IOException { - consumer.closeDocStore(state); - endConsumer.closeDocStore(state); - } - - @Override void abort() { consumer.abort(); endConsumer.abort(); Index: lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (revision ) @@ -79,8 +79,7 @@ int docsMerged = merger.merge(); assertTrue(docsMerged == 2); //Should be able to open a new SegmentReader against the new directory - SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, -1, - null, false, merger.fieldInfos().hasProx(), + SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()), BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (revision ) @@ -47,12 +47,6 @@ } @Override - public void closeDocStore(SegmentWriteState state) throws IOException { - consumer.closeDocStore(state); - fieldsWriter.closeDocStore(state); - } - - @Override public void flush(Collection threads, SegmentWriteState state) throws IOException { Map> childThreadsAndFields = new HashMap>(); Index: lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (revision ) @@ -96,7 +96,7 @@ writer.addDocument(doc); failure.setDoFail(); try { - writer.flush(true, false, true); + writer.flush(true, true); if (failure.hitExc) { fail("failed to hit IOException"); } Index: lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java =================================================================== --- lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java (revision ) @@ -288,7 +288,6 @@ create ? OpenMode.CREATE : OpenMode.APPEND); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundIndex); - lmp.setUseCompoundDocStore(useCompoundIndex); lmp.setMergeFactor(mergeFactor); IndexWriter writer = new IndexWriter(dir, conf); int totalFiles = 0; Index: lucene/src/java/org/apache/lucene/index/NormsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/NormsWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/NormsWriter.java (revision ) @@ -171,7 +171,4 @@ normsOut.close(); } } - - @Override - void closeDocStore(SegmentWriteState state) {} } Index: lucene/src/test/org/apache/lucene/index/TestLazyBug.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLazyBug.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestLazyBug.java (revision ) @@ -66,8 +66,7 @@ TEST_VERSION_CURRENT, new MockAnalyzer())); LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); - + for (int d = 1; d <= NUM_DOCS; d++) { Document doc = new Document(); for (int f = 1; f <= NUM_FIELDS; f++ ) { Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision ) @@ -570,7 +570,6 @@ Directory dir = newDirectory(); LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(100); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()) @@ -599,7 +598,6 @@ lmp = new LogByteSizeMergePolicy(); lmp.setMinMergeMB(0.0001); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(4); writer = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) Index: lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (revision ) @@ -24,7 +24,6 @@ import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; import org.apache.lucene.index.TermsEnum.SeekStatus; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; @@ -73,7 +72,7 @@ // flushing without applying deletes means // there will still be deletes in the segment infos - writer.flush(false, false, false); + writer.flush(false, false); assertTrue(writer.bufferedDeletes.any()); // get reader flushes pending deletes @@ -86,7 +85,7 @@ // merge segments 0 and 1 // which should apply the delete id:2 writer.deleteDocuments(new Term("id", "2")); - writer.flush(false, false, false); + writer.flush(false, false); fsmp.doMerge = true; fsmp.start = 0; fsmp.length = 2; @@ -179,12 +178,12 @@ writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2)); //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs()); } - writer.flush(false, true, false); + writer.flush(false, false); for (int x = 25; x < 30; x++) { writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2)); //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs()); } - writer.flush(false, true, false); + writer.flush(false, false); //System.out.println("infos3:"+writer.segmentInfos); @@ -287,11 +286,6 @@ } @Override - public boolean useCompoundDocStore(SegmentInfos segments) { - return useCompoundFile; - } - - @Override public boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) { return useCompoundFile; } Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision ) @@ -86,22 +86,18 @@ private Map diagnostics; - public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, int docStoreOffset, - String docStoreSegment, boolean docStoreIsCompoundFile, boolean hasProx, SegmentCodecs segmentCodecs, - boolean hasVectors) { + public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, + boolean hasProx, SegmentCodecs segmentCodecs, boolean hasVectors) { this.name = name; this.docCount = docCount; this.dir = dir; delGen = NO; this.isCompoundFile = isCompoundFile; - this.docStoreOffset = docStoreOffset; - this.docStoreSegment = docStoreSegment; - this.docStoreIsCompoundFile = docStoreIsCompoundFile; + this.docStoreOffset = -1; this.hasProx = hasProx; this.segmentCodecs = segmentCodecs; this.hasVectors = hasVectors; delCount = 0; - assert docStoreOffset == -1 || docStoreSegment != null: "dso=" + docStoreOffset + " dss=" + docStoreSegment + " docCount=" + docCount; } /** @@ -255,7 +251,10 @@ @Override public Object clone() { - SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, docStoreOffset, docStoreSegment, docStoreIsCompoundFile, hasProx, segmentCodecs, false); + SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, hasProx, segmentCodecs, false); + si.docStoreOffset = docStoreOffset; + si.docStoreSegment = docStoreSegment; + si.docStoreIsCompoundFile = docStoreIsCompoundFile; si.delGen = delGen; si.delCount = delCount; si.diagnostics = new HashMap(diagnostics); Index: lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java =================================================================== --- lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java (revision ) @@ -251,7 +251,6 @@ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( Version.LUCENE_CURRENT, ana).setOpenMode(OpenMode.CREATE)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why? - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(true); // why? Iterator i1 = word2Nums.keySet().iterator(); while (i1.hasNext()) // for each word { Index: lucene/src/java/org/apache/lucene/index/FieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldsWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/FieldsWriter.java (revision ) @@ -25,9 +25,9 @@ import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IOUtils; -final class FieldsWriter -{ +final class FieldsWriter { static final byte FIELD_IS_TOKENIZED = 0x1; static final byte FIELD_IS_BINARY = 0x2; @@ -41,191 +41,137 @@ // when removing support for old versions, leave the last supported version here static final int FORMAT_MINIMUM = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; - + + // If null - we were supplied with streams, if notnull - we manage them ourselves + private Directory directory; + private String segment; - private FieldInfos fieldInfos; + private FieldInfos fieldInfos; - - private IndexOutput fieldsStream; + private IndexOutput fieldsStream; - - private IndexOutput indexStream; + private IndexOutput indexStream; - private boolean doClose; - - FieldsWriter(Directory d, String segment, FieldInfos fn) throws IOException { + FieldsWriter(Directory directory, String segment, FieldInfos fn) throws IOException { + this.directory = directory; + this.segment = segment; - fieldInfos = fn; + fieldInfos = fn; - boolean success = false; + boolean success = false; - final String fieldsName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION); - try { + try { - fieldsStream = d.createOutput(fieldsName); - fieldsStream.writeInt(FORMAT_CURRENT); - success = true; - } finally { - if (!success) { - try { - close(); - } catch (Throwable t) { - // Suppress so we keep throwing the original exception - } - try { - d.deleteFile(fieldsName); - } catch (Throwable t) { - // Suppress so we keep throwing the original exception - } - } - } + fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); + indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); - success = false; - final String indexName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); - try { - indexStream = d.createOutput(indexName); + fieldsStream.writeInt(FORMAT_CURRENT); - indexStream.writeInt(FORMAT_CURRENT); + indexStream.writeInt(FORMAT_CURRENT); + - success = true; - } finally { + success = true; + } finally { - if (!success) { - try { - close(); - } catch (IOException ioe) { + if (!success) + abort(); - } + } - try { - d.deleteFile(fieldsName); - } catch (Throwable t) { - // Suppress so we keep throwing the original exception - } + } - try { - d.deleteFile(indexName); - } catch (Throwable t) { - // Suppress so we keep throwing the original exception - } - } - } - doClose = true; - } - - FieldsWriter(IndexOutput fdx, IndexOutput fdt, FieldInfos fn) { + FieldsWriter(IndexOutput fdx, IndexOutput fdt, FieldInfos fn) { + directory = null; + segment = null; - fieldInfos = fn; - fieldsStream = fdt; - indexStream = fdx; + fieldInfos = fn; + fieldsStream = fdt; + indexStream = fdx; - doClose = false; - } + } - void setFieldsStream(IndexOutput stream) { - this.fieldsStream = stream; - } + void setFieldsStream(IndexOutput stream) { + this.fieldsStream = stream; + } - // Writes the contents of buffer into the fields stream - // and adds a new entry for this document into the index - // stream. This assumes the buffer was already written - // in the correct fields format. - void flushDocument(int numStoredFields, RAMOutputStream buffer) throws IOException { - indexStream.writeLong(fieldsStream.getFilePointer()); - fieldsStream.writeVInt(numStoredFields); - buffer.writeTo(fieldsStream); - } + // Writes the contents of buffer into the fields stream + // and adds a new entry for this document into the index + // stream. This assumes the buffer was already written + // in the correct fields format. + void flushDocument(int numStoredFields, RAMOutputStream buffer) throws IOException { + indexStream.writeLong(fieldsStream.getFilePointer()); + fieldsStream.writeVInt(numStoredFields); + buffer.writeTo(fieldsStream); + } - void skipDocument() throws IOException { - indexStream.writeLong(fieldsStream.getFilePointer()); - fieldsStream.writeVInt(0); - } + void skipDocument() throws IOException { + indexStream.writeLong(fieldsStream.getFilePointer()); + fieldsStream.writeVInt(0); + } - void flush() throws IOException { - indexStream.flush(); - fieldsStream.flush(); - } - - final void close() throws IOException { - if (doClose) { + void close() throws IOException { + if (directory != null) { - try { + try { - if (fieldsStream != null) { - try { - fieldsStream.close(); + IOUtils.closeSafely(fieldsStream, indexStream); - } finally { + } finally { - fieldsStream = null; + fieldsStream = indexStream = null; - } - } + } + } - } catch (IOException ioe) { - try { - if (indexStream != null) { - try { - indexStream.close(); - } finally { - indexStream = null; - } + } + + void abort() { + if (directory != null) { + try { close(); } catch (IOException ignored) {} + try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); } catch (IOException ignored) {} + try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); } catch (IOException ignored) {} - } + } - } catch (IOException ioe2) { - // Ignore so we throw only first IOException hit - } + } - throw ioe; - } finally { - if (indexStream != null) { - try { - indexStream.close(); - } finally { - indexStream = null; - } - } - } - } - } - final void writeField(FieldInfo fi, Fieldable field) throws IOException { - fieldsStream.writeVInt(fi.number); - byte bits = 0; - if (field.isTokenized()) - bits |= FieldsWriter.FIELD_IS_TOKENIZED; - if (field.isBinary()) - bits |= FieldsWriter.FIELD_IS_BINARY; - - fieldsStream.writeByte(bits); - - if (field.isBinary()) { - final byte[] data; - final int len; - final int offset; - data = field.getBinaryValue(); - len = field.getBinaryLength(); - offset = field.getBinaryOffset(); + final void writeField(FieldInfo fi, Fieldable field) throws IOException { + fieldsStream.writeVInt(fi.number); + byte bits = 0; + if (field.isTokenized()) + bits |= FieldsWriter.FIELD_IS_TOKENIZED; + if (field.isBinary()) + bits |= FieldsWriter.FIELD_IS_BINARY; + + fieldsStream.writeByte(bits); + + if (field.isBinary()) { + final byte[] data; + final int len; + final int offset; + data = field.getBinaryValue(); + len = field.getBinaryLength(); + offset = field.getBinaryOffset(); - fieldsStream.writeVInt(len); - fieldsStream.writeBytes(data, offset, len); - } - else { - fieldsStream.writeString(field.stringValue()); - } - } + fieldsStream.writeVInt(len); + fieldsStream.writeBytes(data, offset, len); + } + else { + fieldsStream.writeString(field.stringValue()); + } + } - /** Bulk write a contiguous series of documents. The - * lengths array is the length (in bytes) of each raw - * document. The stream IndexInput is the - * fieldsStream from which we should bulk-copy all - * bytes. */ - final void addRawDocuments(IndexInput stream, int[] lengths, int numDocs) throws IOException { - long position = fieldsStream.getFilePointer(); - long start = position; - for(int i=0;i fields = doc.getFields(); - for (Fieldable field : fields) { - if (field.isStored()) - storedCount++; - } - fieldsStream.writeVInt(storedCount); + int storedCount = 0; + List fields = doc.getFields(); + for (Fieldable field : fields) { + if (field.isStored()) + storedCount++; + } + fieldsStream.writeVInt(storedCount); - + - for (Fieldable field : fields) { - if (field.isStored()) - writeField(fieldInfos.fieldInfo(field.name()), field); - } - } + for (Fieldable field : fields) { + if (field.isStored()) + writeField(fieldInfos.fieldInfo(field.name()), field); + } + } } Index: lucene/src/test/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (revision ) @@ -688,7 +688,6 @@ public static LogMergePolicy newLogMergePolicy(Random r) { LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); - logmp.setUseCompoundDocStore(r.nextBoolean()); logmp.setUseCompoundFile(r.nextBoolean()); logmp.setCalibrateSizeByDeletes(r.nextBoolean()); if (r.nextInt(3) == 2) { @@ -702,14 +701,12 @@ public static LogMergePolicy newLogMergePolicy(boolean useCFS) { LogMergePolicy logmp = newLogMergePolicy(); logmp.setUseCompoundFile(useCFS); - logmp.setUseCompoundDocStore(useCFS); return logmp; } public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) { LogMergePolicy logmp = newLogMergePolicy(); logmp.setUseCompoundFile(useCFS); - logmp.setUseCompoundDocStore(useCFS); logmp.setMergeFactor(mergeFactor); return logmp; } Index: lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision ) @@ -21,15 +21,8 @@ import java.io.FilenameFilter; import java.io.IOException; import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; +import java.util.*; -import java.util.List; -import java.util.Map; - import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NoSuchDirectoryException; @@ -101,7 +94,6 @@ private PrintStream infoStream; private Directory directory; private IndexDeletionPolicy policy; - private DocumentsWriter docWriter; final boolean startingCommitDeleted; private SegmentInfos lastSegmentInfos; @@ -130,11 +122,7 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter, - CodecProvider codecs) - throws CorruptIndexException, IOException { - - this.docWriter = docWriter; + public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, CodecProvider codecs) throws CorruptIndexException, IOException { this.infoStream = infoStream; final String currentSegmentsFile = segmentInfos.getCurrentSegmentFileName(); @@ -444,31 +432,13 @@ // Decref files for commits that were deleted by the policy: deleteCommits(); } else { - - final List docWriterFiles; - if (docWriter != null) { - docWriterFiles = docWriter.openFiles(); - if (docWriterFiles != null) - // We must incRef these files before decRef'ing - // last files to make sure we don't accidentally - // delete them: - incRef(docWriterFiles); - } else - docWriterFiles = null; - // DecRef old files from the last checkpoint, if any: - int size = lastFiles.size(); - if (size > 0) { - for(int i=0;i lastFile : lastFiles) + decRef(lastFile); - lastFiles.clear(); + lastFiles.clear(); - } // Save files so we can decr on next checkpoint/commit: lastFiles.add(segmentInfos.files(directory, false)); - - if (docWriterFiles != null) - lastFiles.add(docWriterFiles); } } Index: lucene/src/java/org/apache/lucene/index/LogMergePolicy.java =================================================================== --- lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (revision ) @@ -70,7 +70,6 @@ protected boolean calibrateSizeByDeletes = true; protected boolean useCompoundFile = true; - protected boolean useCompoundDocStore = true; public LogMergePolicy() { super(); @@ -157,27 +156,6 @@ return useCompoundFile; } - // Javadoc inherited - @Override - public boolean useCompoundDocStore(SegmentInfos infos) { - return useCompoundDocStore; - } - - /** Sets whether compound file format should be used for - * newly flushed and newly merged doc store - * segment files (term vectors and stored fields). */ - public void setUseCompoundDocStore(boolean useCompoundDocStore) { - this.useCompoundDocStore = useCompoundDocStore; - } - - /** Returns true if newly flushed and newly merge doc - * store segment files (term vectors and stored fields) - * are written in compound file format. @see - * #setUseCompoundDocStore */ - public boolean getUseCompoundDocStore() { - return useCompoundDocStore; - } - /** Sets whether the segment size should be calibrated by * the number of deletes when choosing segments for merge. */ public void setCalibrateSizeByDeletes(boolean calibrateSizeByDeletes) { @@ -591,8 +569,7 @@ sb.append("maxMergeSize=").append(maxMergeSize).append(", "); sb.append("calibrateSizeByDeletes=").append(calibrateSizeByDeletes).append(", "); sb.append("maxMergeDocs=").append(maxMergeDocs).append(", "); - sb.append("useCompoundFile=").append(useCompoundFile).append(", "); - sb.append("useCompoundDocStore=").append(useCompoundDocStore); + sb.append("useCompoundFile=").append(useCompoundFile); sb.append("]"); return sb.toString(); } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision ) @@ -689,7 +689,6 @@ LogMergePolicy lmp = (LogMergePolicy) modifier.getConfig().getMergePolicy(); lmp.setUseCompoundFile(true); - lmp.setUseCompoundDocStore(true); dir.failOn(failure.reset()); Index: lucene/src/test/org/apache/lucene/index/TestOmitTf.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestOmitTf.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestOmitTf.java (revision ) @@ -219,7 +219,6 @@ LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setMergeFactor(2); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); Document d = new Document(); Field f1 = newField("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED); Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision ) @@ -62,12 +62,6 @@ private final CheckAbort checkAbort; - // Whether we should merge doc stores (stored fields and - // vectors files). When all segments we are merging - // already share the same doc store files, we don't need - // to merge the doc stores. - private boolean mergeDocStores; - /** Maximum number of contiguous documents to bulk-copy when merging stored fields */ private final static int MAX_RAW_MERGE_DOCS = 4192; @@ -115,22 +109,6 @@ * @throws IOException if there is a low-level IO error */ final int merge() throws CorruptIndexException, IOException { - return merge(true); - } - - /** - * Merges the readers specified by the {@link #add} method - * into the directory passed to the constructor. - * @param mergeDocStores if false, we will not merge the - * stored fields nor vectors files - * @return The number of documents that were merged - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - final int merge(boolean mergeDocStores) throws CorruptIndexException, IOException { - - this.mergeDocStores = mergeDocStores; - // NOTE: it's important to add calls to // checkAbort.work(...) if you make any changes to this // method that will spend alot of time. The frequency @@ -142,9 +120,8 @@ mergeTerms(); mergeNorms(); - if (mergeDocStores && fieldInfos.hasVectors()) { + if (fieldInfos.hasVectors()) mergeVectors(); - } return mergedDocs; } @@ -154,9 +131,7 @@ // Basic files for (String ext : IndexFileNames.COMPOUND_EXTENSIONS_NOT_CODEC) { - if (mergeDocStores || (!ext.equals(IndexFileNames.FIELDS_EXTENSION) && - !ext.equals(IndexFileNames.FIELDS_INDEX_EXTENSION))) - fileSet.add(IndexFileNames.segmentFileName(segment, "", ext)); + fileSet.add(IndexFileNames.segmentFileName(segment, "", ext)); } segmentWriteState.segmentCodecs.files(directory, info, fileSet); @@ -172,7 +147,7 @@ } // Vector files - if (fieldInfos.hasVectors() && mergeDocStores) { + if (fieldInfos.hasVectors()) { for (String ext : IndexFileNames.VECTOR_EXTENSIONS) { fileSet.add(IndexFileNames.segmentFileName(segment, "", ext)); } @@ -251,18 +226,8 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private final int mergeFields() throws CorruptIndexException, IOException { - - if (!mergeDocStores) { - // When we are not merging by doc stores, their field - // name -> number mapping are the same. So, we start - // with the fieldInfos of the last segment in this - // case, to keep that numbering. - final SegmentReader sr = (SegmentReader) readers.get(readers.size()-1); - fieldInfos = (FieldInfos) sr.core.fieldInfos.clone(); - } else { + private int mergeFields() throws CorruptIndexException, IOException { - fieldInfos = new FieldInfos();// merge field names + fieldInfos = new FieldInfos();// merge field names - } for (IndexReader reader : readers) { if (reader instanceof SegmentReader) { @@ -294,55 +259,44 @@ setMatchingSegmentReaders(); - if (mergeDocStores) { - // merge field values - final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos); + final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos); - try { - int idx = 0; - for (IndexReader reader : readers) { - final SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++]; - FieldsReader matchingFieldsReader = null; - if (matchingSegmentReader != null) { - final FieldsReader fieldsReader = matchingSegmentReader.getFieldsReader(); - if (fieldsReader != null) { - matchingFieldsReader = fieldsReader; - } - } - if (reader.hasDeletions()) { - docCount += copyFieldsWithDeletions(fieldsWriter, - reader, matchingFieldsReader); - } else { - docCount += copyFieldsNoDeletions(fieldsWriter, - reader, matchingFieldsReader); - } - } - } finally { - fieldsWriter.close(); - } + try { + int idx = 0; + for (IndexReader reader : readers) { + final SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++]; + FieldsReader matchingFieldsReader = null; + if (matchingSegmentReader != null) { + final FieldsReader fieldsReader = matchingSegmentReader.getFieldsReader(); + if (fieldsReader != null) { + matchingFieldsReader = fieldsReader; + } + } + if (reader.hasDeletions()) { + docCount += copyFieldsWithDeletions(fieldsWriter, + reader, matchingFieldsReader); + } else { + docCount += copyFieldsNoDeletions(fieldsWriter, + reader, matchingFieldsReader); + } + } + } finally { + fieldsWriter.close(); + } - final String fileName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); - final long fdxFileLength = directory.fileLength(fileName); + final String fileName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + final long fdxFileLength = directory.fileLength(fileName); - if (4+((long) docCount)*8 != fdxFileLength) - // This is most likely a bug in Sun JRE 1.6.0_04/_05; - // we detect that the bug has struck, here, and - // throw an exception to prevent the corruption from - // entering the index. See LUCENE-1282 for - // details. - throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); + if (4+((long) docCount)*8 != fdxFileLength) + // This is most likely a bug in Sun JRE 1.6.0_04/_05; + // we detect that the bug has struck, here, and + // throw an exception to prevent the corruption from + // entering the index. See LUCENE-1282 for + // details. + throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - } else { - // If we are skipping the doc stores, that means there - // are no deletions in any of these segments, so we - // just sum numDocs() of each segment to get total docCount - for (final IndexReader reader : readers) { - docCount += reader.numDocs(); - } - } + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo); - + - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, null, docCount, 0, termIndexInterval, codecInfo); - return docCount; } Index: lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision ) @@ -56,7 +56,6 @@ addDoc(writer, i); } mergePolicy.setUseCompoundFile(false); - mergePolicy.setUseCompoundDocStore(false); for(;i<45;i++) { addDoc(writer, i); } Index: lucene/src/test/org/apache/lucene/index/TestFieldsReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (revision ) @@ -53,7 +53,6 @@ dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()); ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(false); IndexWriter writer = new IndexWriter(dir, conf); writer.addDocument(testDoc); writer.close(); Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision ) @@ -240,7 +240,6 @@ LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(true); - lmp.setUseCompoundDocStore(true); iw.close(); } @@ -293,7 +292,6 @@ LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(compound); - lmp.setUseCompoundDocStore(compound); IndexWriter iw = new IndexWriter(dir, conf); for (int i = 0; i < ndocs; i++) { iw.addDocument(newDoc()); Index: lucene/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DirectoryReader.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/DirectoryReader.java (revision ) @@ -765,7 +765,7 @@ // KeepOnlyLastCommitDeleter: IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, null, null, codecs); + segmentInfos, null, codecs); segmentInfos.updateGeneration(deleter.getLastSegmentInfos()); segmentInfos.changed(); Index: lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision ) @@ -39,9 +39,6 @@ } @Override - void closeDocStore(SegmentWriteState state) {} - - @Override void abort() {} private int flushedDocCount; Index: lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (revision ) @@ -154,7 +154,6 @@ w.commit(); LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(mergeFactor); /*** w.setMaxMergeDocs(Integer.MAX_VALUE); @@ -211,7 +210,6 @@ w.setInfoStream(VERBOSE ? System.out : null); LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(mergeFactor); threads = new IndexingThread[nThreads]; Index: lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java (revision ) @@ -35,16 +35,13 @@ assertNull(mp.findMerges(null)); assertNull(mp.findMergesForOptimize(null, 0, null)); assertNull(mp.findMergesToExpungeDeletes(null)); - assertFalse(mp.useCompoundDocStore(null)); assertFalse(mp.useCompoundFile(null, null)); mp.close(); } @Test public void testCompoundFiles() throws Exception { - assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundDocStore(null)); assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundFile(null, null)); - assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundDocStore(null)); assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundFile(null, null)); } Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 6f46a19392a836e5ff174bfdbb3639ab6febf6af) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision ) @@ -346,7 +346,7 @@ // just like we do when loading segments_N IndexReader r; synchronized(this) { - flush(false, true, true); + flush(false, true); r = new DirectoryReader(this, segmentInfos, config.getReaderTermsIndexDivisor(), codecs); if (infoStream != null) { message("return reader version=" + r.getVersion() + " reader=" + r); @@ -773,7 +773,7 @@ // KeepOnlyLastCommitDeleter: deleter = new IndexFileDeleter(directory, conf.getIndexDeletionPolicy(), - segmentInfos, infoStream, docWriter, codecs); + segmentInfos, infoStream, codecs); if (deleter.startingCommitDeleted) { // Deletion policy deleted the "head" commit point. @@ -1037,7 +1037,7 @@ // Only allow a new merge to be triggered if we are // going to wait for merges: if (!hitOOM) { - flush(waitForMerges, true, true); + flush(waitForMerges, true); } if (waitForMerges) @@ -1111,7 +1111,7 @@ public synchronized int maxDoc() { int count; if (docWriter != null) - count = docWriter.getNumDocsInRAM(); + count = docWriter.getNumDocs(); else count = 0; @@ -1129,7 +1129,7 @@ public synchronized int numDocs() throws IOException { int count; if (docWriter != null) - count = docWriter.getNumDocsInRAM(); + count = docWriter.getNumDocs(); else count = 0; @@ -1253,18 +1253,12 @@ synchronized (this) { // If docWriter has some aborted files that were // never incref'd, then we clean them up here - deleter.checkpoint(segmentInfos, false); - if (docWriter != null) { - final Collection files = docWriter.abortedFiles(); - if (files != null) { - deleter.deleteNewFiles(files); + deleter.checkpoint(segmentInfos, false); // nocommit: is this needed? - } - } - } + } + } + } - } - } if (doFlush) - flush(true, false, false); + flush(true, false); } catch (OutOfMemoryError oom) { handleOOM(oom, "addDocument"); } @@ -1285,7 +1279,7 @@ ensureOpen(); try { if (docWriter.deleteTerm(term, false)) { - flush(true, false, false); + flush(true, false); } } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteDocuments(Term)"); @@ -1309,7 +1303,7 @@ ensureOpen(); try { if (docWriter.deleteTerms(terms)) { - flush(true, false, false); + flush(true, false); } } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteDocuments(Term..)"); @@ -1331,7 +1325,7 @@ ensureOpen(); try { if (docWriter.deleteQuery(query)) { - flush(true, false, false); + flush(true, false); } } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteDocuments(Query)"); @@ -1355,7 +1349,7 @@ ensureOpen(); try { if (docWriter.deleteQueries(queries)) { - flush(true, false, false); + flush(true, false); } } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteDocuments(Query..)"); @@ -1412,26 +1406,11 @@ doFlush = docWriter.updateDocument(doc, analyzer, term); success = true; } finally { - if (!success) { - - if (infoStream != null) { + if (!success && infoStream != null) - message("hit exception updating document"); - } + message("hit exception updating document"); + } - - synchronized (this) { - // If docWriter has some aborted files that were - // never incref'd, then we clean them up here - if (docWriter != null) { - final Collection files = docWriter.abortedFiles(); - if (files != null) { - deleter.deleteNewFiles(files); - } - } - } - } - } if (doFlush) { - flush(true, false, false); + flush(true, false); } } catch (OutOfMemoryError oom) { handleOOM(oom, "updateDocument"); @@ -1445,7 +1424,7 @@ // for test purpose final synchronized int getNumBufferedDocuments(){ - return docWriter.getNumDocsInRAM(); + return docWriter.getNumDocs(); } // for test purpose @@ -1601,7 +1580,7 @@ message("now flush at optimize"); } - flush(true, false, true); + flush(true, true); synchronized(this) { resetMergeExceptions(); @@ -2141,7 +2120,7 @@ try { if (infoStream != null) message("flush at addIndexes(Directory...)"); - flush(false, false, true); + flush(false, true); int docCount = 0; List infos = new ArrayList(); @@ -2241,7 +2220,7 @@ int docCount = merger.merge(); // merge 'em SegmentInfo info = new SegmentInfo(mergedName, docCount, directory, - false, -1, null, false, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(), + false, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()); setDiagnostics(info, "addIndexes(IndexReader...)"); @@ -2337,7 +2316,7 @@ if (infoStream != null) message("prepareCommit: flush"); - flush(true, true, true); + flush(true, true); startCommit(commitUserData); } @@ -2454,12 +2433,9 @@ * to the Directory. * @param triggerMerge if true, we may merge segments (if * deletes or docs were flushed) if necessary - * @param flushDocStores if false we are allowed to keep - * doc stores open to share with the next segment * @param flushDeletes whether pending deletes should also - * be flushed */ - protected final void flush(boolean triggerMerge, boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException { + protected final void flush(boolean triggerMerge, boolean flushDeletes) throws CorruptIndexException, IOException { // NOTE: this method cannot be sync'd because // maybeMerge() in turn calls mergeScheduler.merge which @@ -2470,7 +2446,7 @@ // We can be called during close, when closing==true, so we must pass false to ensureOpen: ensureOpen(false); - if (doFlush(flushDocStores, flushDeletes) && triggerMerge) { + if (doFlush(flushDeletes) && triggerMerge) { maybeMerge(); } } @@ -2478,7 +2454,7 @@ // TODO: this method should not have to be entirely // synchronized, ie, merges should be allowed to commit // even while a flush is happening - private synchronized final boolean doFlush(boolean closeDocStores, boolean applyAllDeletes) throws CorruptIndexException, IOException { + private synchronized boolean doFlush(boolean applyAllDeletes) throws CorruptIndexException, IOException { if (hitOOM) { throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot flush"); @@ -2501,11 +2477,11 @@ try { if (infoStream != null) { - message(" start flush: applyAllDeletes=" + applyAllDeletes + " closeDocStores=" + closeDocStores); + message(" start flush: applyAllDeletes=" + applyAllDeletes); message(" index before flush " + segString()); } - final SegmentInfo newSegment = docWriter.flush(this, closeDocStores, deleter, mergePolicy, segmentInfos); + final SegmentInfo newSegment = docWriter.flush(this, deleter, mergePolicy, segmentInfos); if (newSegment != null) { setDiagnostics(newSegment, "flush"); segmentInfos.add(newSegment); @@ -2553,19 +2529,10 @@ return false; } finally { flushControl.clearFlushPending(); - if (!success) { - if (infoStream != null) { + if (!success && infoStream != null) - message("hit exception during flush"); - } + message("hit exception during flush"); + } - if (docWriter != null) { - final Collection files = docWriter.abortedFiles(); - if (files != null) { - deleter.deleteNewFiles(files); - } + } - } - } - } - } /** Expert: Return the total size of all index files currently cached in memory. * Useful for size management with flushRamDocs() @@ -2579,7 +2546,7 @@ * buffered in RAM. */ public final synchronized int numRamDocs() { ensureOpen(); - return docWriter.getNumDocsInRAM(); + return docWriter.getNumDocs(); } private int ensureContiguousMerge(MergePolicy.OneMerge merge) { @@ -2916,7 +2883,7 @@ } } - final synchronized private void _mergeInit(MergePolicy.OneMerge merge) throws IOException { + synchronized private void _mergeInit(MergePolicy.OneMerge merge) throws IOException { assert testPoint("startMergeInit"); @@ -2933,133 +2900,26 @@ if (merge.isAborted()) return; - + - final SegmentInfos sourceSegments = merge.segments; - final int end = sourceSegments.size(); - - // Check whether this merge will allow us to skip - // merging the doc stores (stored field & vectors). - // This is a very substantial optimization (saves tons - // of IO). - - Directory lastDir = directory; - String lastDocStoreSegment = null; - int next = -1; - - boolean mergeDocStores = false; - boolean doFlushDocStore = false; boolean hasVectors = false; - final String currentDocStoreSegment = docWriter.getDocStoreSegment(); - - // Test each segment to be merged: check if we need to - // flush/merge doc stores - for (int i = 0; i < end; i++) { - SegmentInfo si = sourceSegments.info(i); - - // If it has deletions we must merge the doc stores - if (si.hasDeletions()) - mergeDocStores = true; - - if (si.getHasVectors()) { + for (SegmentInfo sourceSegment : merge.segments) { + if (sourceSegment.getHasVectors()) { hasVectors = true; } - - // If it has its own (private) doc stores we must - // merge the doc stores - if (-1 == si.getDocStoreOffset()) - mergeDocStores = true; - - // If it has a different doc store segment than - // previous segments, we must merge the doc stores - String docStoreSegment = si.getDocStoreSegment(); - if (docStoreSegment == null) - mergeDocStores = true; - else if (lastDocStoreSegment == null) - lastDocStoreSegment = docStoreSegment; - else if (!lastDocStoreSegment.equals(docStoreSegment)) - mergeDocStores = true; - - // Segments' docScoreOffsets must be in-order, - // contiguous. For the default merge policy now - // this will always be the case but for an arbitrary - // merge policy this may not be the case - if (-1 == next) - next = si.getDocStoreOffset() + si.docCount; - else if (next != si.getDocStoreOffset()) - mergeDocStores = true; - else - next = si.getDocStoreOffset() + si.docCount; - - // If the segment comes from a different directory - // we must merge - if (lastDir != si.dir) - mergeDocStores = true; - - // If the segment is referencing the current "live" - // doc store outputs then we must merge - if (si.getDocStoreOffset() != -1 && currentDocStoreSegment != null && si.getDocStoreSegment().equals(currentDocStoreSegment)) { - doFlushDocStore = true; - } + } - } - // if a mergedSegmentWarmer is installed, we must merge - // the doc stores because we will open a full - // SegmentReader on the merged segment: - if (!mergeDocStores && mergedSegmentWarmer != null && currentDocStoreSegment != null && lastDocStoreSegment != null && lastDocStoreSegment.equals(currentDocStoreSegment)) { - mergeDocStores = true; - } - - final int docStoreOffset; - final String docStoreSegment; - final boolean docStoreIsCompoundFile; - - if (mergeDocStores) { - docStoreOffset = -1; - docStoreSegment = null; - docStoreIsCompoundFile = false; - } else { - SegmentInfo si = sourceSegments.info(0); - docStoreOffset = si.getDocStoreOffset(); - docStoreSegment = si.getDocStoreSegment(); - docStoreIsCompoundFile = si.getDocStoreIsCompoundFile(); - } - - if (mergeDocStores && doFlushDocStore) { - // SegmentMerger intends to merge the doc stores - // (stored fields, vectors), and at least one of the - // segments to be merged refers to the currently - // live doc stores. - - // TODO: if we know we are about to merge away these - // newly flushed doc store files then we should not - // make compound file out of them... - if (infoStream != null) - message("now flush at merge"); - doFlush(true, false); - updatePendingMerges(1, false); - } - - merge.mergeDocStores = mergeDocStores; - // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment // names. - merge.info = new SegmentInfo(newSegmentName(), 0, - directory, false, docStoreOffset, - docStoreSegment, - docStoreIsCompoundFile, - false, - null, - hasVectors); + merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, false, null, hasVectors); Map details = new HashMap(); details.put("optimize", Boolean.toString(merge.optimize)); - details.put("mergeFactor", Integer.toString(end)); - details.put("mergeDocStores", Boolean.toString(mergeDocStores)); + details.put("mergeFactor", Integer.toString(merge.segments.size())); setDiagnostics(merge.info, "merge", details); if (infoStream != null) { - message("merge seg=" + merge.info.name + " mergeDocStores=" + mergeDocStores); + message("merge seg=" + merge.info.name); } // Also enroll the merged segment into mergingSegments; @@ -3183,8 +3043,6 @@ final String mergedName = merge.info.name; - SegmentMerger merger = null; - int mergedDocCount = 0; SegmentInfos sourceSegments = merge.segments; @@ -3193,20 +3051,11 @@ if (infoStream != null) message("merging " + merge.segString(directory)); - merger = new SegmentMerger(directory, termIndexInterval, mergedName, merge, codecs, payloadProcessorProvider); + SegmentMerger merger = new SegmentMerger(directory, termIndexInterval, mergedName, merge, codecs, payloadProcessorProvider); merge.readers = new SegmentReader[numSegments]; merge.readersClone = new SegmentReader[numSegments]; - boolean mergeDocStores = false; - - final String currentDocStoreSegment; - synchronized(this) { - currentDocStoreSegment = docWriter.getDocStoreSegment(); - } - - boolean currentDSSMerged = false; - // This is try/finally to make sure merger's readers are // closed: boolean success = false; @@ -3218,7 +3067,7 @@ // Hold onto the "live" reader; we will use this to // commit merged deletes - SegmentReader reader = merge.readers[i] = readerPool.get(info, merge.mergeDocStores, + SegmentReader reader = merge.readers[i] = readerPool.get(info, true, MERGE_READ_BUFFER_SIZE, -config.getReaderTermsIndexDivisor()); @@ -3228,14 +3077,6 @@ SegmentReader clone = merge.readersClone[i] = (SegmentReader) reader.clone(true); merger.add(clone); - if (clone.hasDeletions()) { - mergeDocStores = true; - } - - if (info.getDocStoreOffset() != -1 && currentDocStoreSegment != null) { - currentDSSMerged |= currentDocStoreSegment.equals(info.getDocStoreSegment()); - } - totDocCount += clone.numDocs(); } @@ -3245,39 +3086,8 @@ merge.checkAborted(directory); - // If deletions have arrived and it has now become - // necessary to merge doc stores, go and open them: - if (mergeDocStores && !merge.mergeDocStores) { - merge.mergeDocStores = true; - synchronized(this) { - - // If 1) we must now merge doc stores, and 2) at - // least one of the segments we are merging uses - // the doc store we are now writing to, we must at - // this point force this doc store closed (by - // calling flush). If we didn't do this then the - // readers will attempt to open an IndexInput - // on files that have still-open IndexOutputs - // against them: - if (currentDSSMerged) { - if (infoStream != null) { - message("now flush at mergeMiddle"); - } - doFlush(true, false); - updatePendingMerges(1, false); - } - } - - for(int i=0;i