Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1491455) +++ lucene/CHANGES.txt (working copy) @@ -225,6 +225,13 @@ JRE vendor via Runtime.halt(). (Mike McCandless, Robert Muir, Uwe Schindler, Rodrigo Trujillo, Dawid Weiss) +Changes in runtime behavior + +* LUCENE-5038: New segments written by IndexWriter are now wrapped into CFS + by default. DocumentsWriterPerThread doesn't consult MergePolicy anymore + to decide if a CFS must be written, instead IndexWriterConfig now has a + property to enable / disable CFS for newly created segments. (Simon Willnauer) + ======================= Lucene 4.3.1 ======================= Bug Fixes Index: lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java =================================================================== --- lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java (revision 1491455) +++ lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java (working copy) @@ -54,7 +54,7 @@ IndexWriter writer = new IndexWriter( fsd, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). - setMergePolicy(newLogMergePolicy(false)).setCodec(Codec.forName("Lucene40")) + setMergePolicy(newLogMergePolicy(false)).setCodec(Codec.forName("Lucene40")).setDocumentsWriterUseCompoundFile(false) ); TestIndexWriterReader.createIndexNoClose(true, "ram", writer); IndexReader reader = DirectoryReader.open(writer, true); Index: lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (revision 1491455) +++ lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (working copy) @@ -49,6 +49,7 @@ iwConf.setCodec(CompressingCodec.randomInstance(random())); // disable CFS because this test checks file names iwConf.setMergePolicy(newLogMergePolicy(false)); + iwConf.setDocumentsWriterUseCompoundFile(false); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf); final Document validDoc = new Document(); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1491455) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -1142,7 +1142,7 @@ writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). - setMergePolicy(newLogMergePolicy(true)) + setMergePolicy(newLogMergePolicy(true)).setDocumentsWriterUseCompoundFile(true) ); LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); // Force creation of CFS: Index: lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 1491455) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (working copy) @@ -54,7 +54,7 @@ dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). setMaxBufferedDocs(10). - setMergePolicy(mergePolicy) + setMergePolicy(mergePolicy).setDocumentsWriterUseCompoundFile(true) ); int i; @@ -62,6 +62,7 @@ addDoc(writer, i); } ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); + writer.getConfig().setDocumentsWriterUseCompoundFile(false); for(;i<45;i++) { addDoc(writer, i); } @@ -71,7 +72,7 @@ writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). - setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES) + setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setDocumentsWriterUseCompoundFile(true) ); Term searchTerm = new Term("id", "7"); writer.deleteDocuments(searchTerm); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1491455) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -1348,7 +1348,7 @@ IndexWriter w = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). - setMergePolicy(mergePolicy) + setMergePolicy(mergePolicy).setDocumentsWriterUseCompoundFile(true) ); Document doc = new Document(); doc.add(newTextField("field", "go", Field.Store.NO)); @@ -1468,7 +1468,7 @@ Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())) - .setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy())); + .setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy()).setDocumentsWriterUseCompoundFile(false)); String[] files = dir.listAll(); // Creating over empty dir should not create any files, Index: lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (revision 1491455) +++ lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (working copy) @@ -93,7 +93,7 @@ dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MyAnalyzer()). setMaxBufferedDocs(-1). - setMergePolicy(newLogMergePolicy(false, 10)) + setMergePolicy(newLogMergePolicy(false, 10)).setDocumentsWriterUseCompoundFile(false) ); Document doc = new Document(); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (revision 1491455) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (working copy) @@ -78,6 +78,7 @@ assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB()); assertEquals(Codec.getDefault(), conf.getCodec()); assertEquals(InfoStream.getDefault(), conf.getInfoStream()); + assertEquals(IndexWriterConfig.DEFAULT_DOCUMENTS_WRITER_USE_CFS, conf.getDocumentsWriterUseCompoundFile()); // Sanity check - validate that all getters are covered. Set getters = new HashSet(); getters.add("getAnalyzer"); @@ -104,6 +105,7 @@ getters.add("getRAMPerThreadHardLimitMB"); getters.add("getCodec"); getters.add("getInfoStream"); + getters.add("getDocumentsWriterUseCompoundFile"); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { @@ -188,6 +190,7 @@ assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS); assertEquals(16.0, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, 0.0); assertEquals(false, IndexWriterConfig.DEFAULT_READER_POOLING); + assertEquals(true, IndexWriterConfig.DEFAULT_DOCUMENTS_WRITER_USE_CFS); assertEquals(DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR); } Index: lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (revision 1491455) +++ lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (working copy) @@ -110,6 +110,10 @@ * others to finish. Default value is 8. */ public final static int DEFAULT_MAX_THREAD_STATES = 8; + /** Default value for compound file system for newly written segments + * (set to true). For batch indexing with very large + * ram buffers use false */ + public final static boolean DEFAULT_DOCUMENTS_WRITER_USE_CFS = true; /** * Sets the default (for any instance) maximum time to wait for a write lock * (in milliseconds). @@ -540,5 +544,9 @@ public IndexWriterConfig setTermIndexInterval(int interval) { return (IndexWriterConfig) super.setTermIndexInterval(interval); } + + public IndexWriterConfig setDocumentsWriterUseCompoundFile(boolean useCompoundFile) { + return (IndexWriterConfig) super.setDocumentsWriterUseCompoundFile(useCompoundFile); + } } Index: lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (revision 1491455) +++ lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (working copy) @@ -98,6 +98,9 @@ /** {@link Version} that {@link IndexWriter} should emulate. */ protected final Version matchVersion; + /** True if segment flushes should use compound file format */ + protected volatile boolean useCompoundFile = IndexWriterConfig.DEFAULT_DOCUMENTS_WRITER_USE_CFS; + // used by IndexWriterConfig LiveIndexWriterConfig(Analyzer analyzer, Version matchVersion) { this.analyzer = analyzer; @@ -110,6 +113,7 @@ termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here delPolicy = new KeepOnlyLastCommitDeletionPolicy(); commit = null; + useCompoundFile = IndexWriterConfig.DEFAULT_DOCUMENTS_WRITER_USE_CFS; openMode = OpenMode.CREATE_OR_APPEND; similarity = IndexSearcher.getDefaultSimilarity(); mergeScheduler = new ConcurrentMergeScheduler(); @@ -154,6 +158,7 @@ readerPooling = config.getReaderPooling(); flushPolicy = config.getFlushPolicy(); perThreadHardLimitMB = config.getRAMPerThreadHardLimitMB(); + useCompoundFile = config.getDocumentsWriterUseCompoundFile(); } /** Returns the default analyzer to use for indexing documents. */ @@ -542,6 +547,27 @@ return infoStream; } + /** + * Sets if the {@link IndexWriter} should pack newly written segments in a + * compound file. Default is true. + *

+ * Use false for batch indexing with very large ram buffer + * settings. + *

+ */ + public LiveIndexWriterConfig setDocumentsWriterUseCompoundFile(boolean useCompoundFile) { + this.useCompoundFile = useCompoundFile; + return this; + } + + /** + * Retruns true iff the {@link IndexWriter} packs + * newly written segments in a compound file. Default is true. + */ + public boolean getDocumentsWriterUseCompoundFile() { + return useCompoundFile ; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -567,7 +593,10 @@ sb.append("indexerThreadPool=").append(getIndexerThreadPool()).append("\n"); sb.append("readerPooling=").append(getReaderPooling()).append("\n"); sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n"); + sb.append("documentsWriterUseCompoundFile=").append(getDocumentsWriterUseCompoundFile()).append("\n"); return sb.toString(); } + + } Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (revision 1491455) +++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (working copy) @@ -194,6 +194,7 @@ private final NumberFormat nf = NumberFormat.getInstance(Locale.ROOT); final Allocator byteBlockAllocator; final IntBlockPool.Allocator intBlockAllocator; + private final LiveIndexWriterConfig indexWriterConfig; public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent, @@ -203,6 +204,7 @@ this.parent = parent; this.fieldInfos = fieldInfos; this.writer = parent.indexWriter; + this.indexWriterConfig = parent.indexWriterConfig; this.infoStream = parent.infoStream; this.codec = parent.codec; this.docState = new DocState(this, infoStream); @@ -567,7 +569,7 @@ boolean success = false; try { - if (writer.useCompoundFile(newSegment)) { + if (indexWriterConfig.getDocumentsWriterUseCompoundFile()) { // Now build compound file Collection oldFiles = IndexWriter.createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, newSegment.info, context); Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (revision 1491455) +++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -29,15 +29,11 @@ import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; import org.apache.lucene.index.FieldInfos.FieldNumbers; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FlushInfo; -import org.apache.lucene.store.IOContext; import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.MutableBits; /** * This class accepts multiple added documents and directly @@ -114,6 +110,7 @@ List newFiles; final IndexWriter indexWriter; + final LiveIndexWriterConfig indexWriterConfig; private AtomicInteger numDocsInRAM = new AtomicInteger(0); @@ -144,6 +141,7 @@ this.indexWriter = writer; this.infoStream = config.getInfoStream(); this.similarity = config.getSimilarity(); + this.indexWriterConfig = writer.getConfig(); this.perThreadPool = config.getIndexerThreadPool(); this.chain = config.getIndexingChain(); this.perThreadPool.initialize(this, globalFieldNumbers, config); @@ -517,7 +515,7 @@ // buffer, force them all to apply now. This is to // prevent too-frequent flushing of a long tail of // tiny segments: - final double ramBufferSizeMB = indexWriter.getConfig().getRAMBufferSizeMB(); + final double ramBufferSizeMB = indexWriterConfig.getRAMBufferSizeMB(); if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH && flushControl.getDeleteBytesUsed() > (1024*1024*ramBufferSizeMB/2)) { if (infoStream.isEnabled("DW")) { Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1491455) +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -2269,10 +2269,6 @@ } } - synchronized boolean useCompoundFile(SegmentInfoPerCommit segmentInfo) throws IOException { - return mergePolicy.useCompoundFile(segmentInfos, segmentInfo); - } - private synchronized void resetMergeExceptions() { mergeExceptions = new ArrayList(); mergeGen++; Index: lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (revision 1491455) +++ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -808,6 +808,7 @@ if (rarely(r)) { c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream())); } + c.setDocumentsWriterUseCompoundFile(r.nextBoolean()); c.setReaderPooling(r.nextBoolean()); c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4)); return c;