Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 932917) +++ lucene/CHANGES.txt (working copy) @@ -105,6 +105,12 @@ of incrementToken(), tokenStream(), and reusableTokenStream(). (Uwe Schindler, Robert Muir) +* LUCENE-2386: IndexWriter no longer performs an empty commit upon new index + creation. Previously, if you passed an empty Directory and set OpenMode to + CREATE*, IndexWriter would make a first empty commit. If you need that + behavior you can call writer.commit() immediately after you create it. + (Shai Erera, Mike McCandless) + Changes in runtime behavior * LUCENE-1923: Made IndexReader.toString() produce something Index: lucene/backwards/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java (working copy) @@ -113,7 +113,7 @@ SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); final IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED); - + writer.commit(); // Force frequent flushes writer.setMaxBufferedDocs(2); Index: lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -553,7 +553,7 @@ expected = new String[] {"_0.cfs", "_0_1.del", "_0_1.s" + contentFieldIndex, - "segments_3", + "segments_2", "segments.gen"}; String[] actual = dir.listAll(); Index: lucene/backwards/src/test/org/apache/lucene/index/TestCrash.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestCrash.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestCrash.java (working copy) @@ -25,20 +25,24 @@ import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; public class TestCrash extends LuceneTestCase { - private IndexWriter initIndex() throws IOException { - return initIndex(new MockRAMDirectory()); + private IndexWriter initIndex(boolean initialCommit) throws IOException { + return initIndex(new MockRAMDirectory(), initialCommit); } - private IndexWriter initIndex(MockRAMDirectory dir) throws IOException { + private IndexWriter initIndex(MockRAMDirectory dir, boolean initialCommit) throws IOException { dir.setLockFactory(NoLockFactory.getNoLockFactory()); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); //writer.setMaxBufferedDocs(2); writer.setMaxBufferedDocs(10); ((ConcurrentMergeScheduler) writer.getMergeScheduler()).setSuppressExceptions(); + if (initialCommit) { + writer.commit(); + } Document doc = new Document(); doc.add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED)); @@ -58,7 +62,7 @@ } public void testCrashWhileIndexing() throws IOException { - IndexWriter writer = initIndex(); + IndexWriter writer = initIndex(true); MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory(); crash(writer); IndexReader reader = IndexReader.open(dir, false); @@ -66,11 +70,11 @@ } public void testWriterAfterCrash() throws IOException { - IndexWriter writer = initIndex(); + IndexWriter writer = initIndex(true); MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory(); dir.setPreventDoubleWrite(false); crash(writer); - writer = initIndex(dir); + writer = initIndex(dir, false); writer.close(); IndexReader reader = IndexReader.open(dir, false); @@ -78,10 +82,10 @@ } public void testCrashAfterReopen() throws IOException { - IndexWriter writer = initIndex(); + IndexWriter writer = initIndex(false); MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory(); writer.close(); - writer = initIndex(dir); + writer = initIndex(dir, false); assertEquals(314, writer.maxDoc()); crash(writer); @@ -100,7 +104,7 @@ public void testCrashAfterClose() throws IOException { - IndexWriter writer = initIndex(); + IndexWriter writer = initIndex(false); MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory(); writer.close(); @@ -119,7 +123,7 @@ public void testCrashAfterCloseNoWait() throws IOException { - IndexWriter writer = initIndex(); + IndexWriter writer = initIndex(false); MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory(); writer.close(false); @@ -138,7 +142,7 @@ public void testCrashReaderDeletes() throws IOException { - IndexWriter writer = initIndex(); + IndexWriter writer = initIndex(false); MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory(); writer.close(false); @@ -159,7 +163,7 @@ public void testCrashReaderDeletesAfterClose() throws IOException { - IndexWriter writer = initIndex(); + IndexWriter writer = initIndex(false); MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory(); writer.close(false); Index: lucene/backwards/src/test/org/apache/lucene/index/TestDeletionPolicy.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestDeletionPolicy.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestDeletionPolicy.java (working copy) @@ -290,7 +290,7 @@ writer.optimize(); writer.close(); - assertEquals(2, policy.numOnInit); + assertEquals(1, policy.numOnInit); // If we are not auto committing then there should // be exactly 2 commits (one per close above): @@ -298,8 +298,8 @@ // Test listCommits Collection commits = IndexReader.listCommits(dir); - // 1 from opening writer + 2 from closing writer - assertEquals(3, commits.size()); + // 2 from closing writer + assertEquals(2, commits.size()); Iterator it = commits.iterator(); // Make sure we can open a reader on each commit: @@ -357,7 +357,7 @@ writer.close(); Collection commits = IndexReader.listCommits(dir); - assertEquals(6, commits.size()); + assertEquals(5, commits.size()); IndexCommit lastCommit = null; Iterator it = commits.iterator(); while(it.hasNext()) { @@ -374,7 +374,7 @@ writer.optimize(); writer.close(); - assertEquals(7, IndexReader.listCommits(dir).size()); + assertEquals(6, IndexReader.listCommits(dir).size()); // Now open writer on the commit just before optimize: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED, lastCommit); @@ -395,7 +395,7 @@ writer.close(); // Now 8 because we made another commit - assertEquals(8, IndexReader.listCommits(dir).size()); + assertEquals(7, IndexReader.listCommits(dir).size()); r = IndexReader.open(dir, true); // Not optimized because we rolled it back, and now only @@ -465,7 +465,7 @@ writer.optimize(); writer.close(); - assertEquals(2, policy.numOnInit); + assertEquals(1, policy.numOnInit); // If we are not auto committing then there should // be exactly 2 commits (one per close above): assertEquals(2, policy.numOnCommit); @@ -506,7 +506,7 @@ } assertTrue(policy.numDelete > 0); - assertEquals(N+1, policy.numOnInit); + assertEquals(N, policy.numOnInit); assertEquals(N+1, policy.numOnCommit); // Simplistic check: just verify only the past N segments_N's still @@ -580,8 +580,8 @@ // this is a commit writer.close(); - assertEquals(2*(N+2), policy.numOnInit); - assertEquals(2*(N+2)-1, policy.numOnCommit); + assertEquals(2*(N+1)+1, policy.numOnInit); + assertEquals(2*(N+2), policy.numOnCommit); IndexSearcher searcher = new IndexSearcher(dir, false); ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; @@ -678,8 +678,8 @@ writer.close(); } - assertEquals(1+3*(N+1), policy.numOnInit); - assertEquals(3*(N+1), policy.numOnCommit); + assertEquals(3*(N+1), policy.numOnInit); + assertEquals(3*(N+1)+1, policy.numOnCommit); IndexSearcher searcher = new IndexSearcher(dir, false); ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexFileDeleter.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (working copy) @@ -136,11 +136,11 @@ copyFile(dir, "_0.cfs", "deletable"); // Create some old segments file: - copyFile(dir, "segments_3", "segments"); - copyFile(dir, "segments_3", "segments_2"); + copyFile(dir, "segments_2", "segments"); + copyFile(dir, "segments_2", "segments_1"); // Create a bogus cfs file shadowing a non-cfs segment: - copyFile(dir, "_2.cfs", "_3.cfs"); + copyFile(dir, "_1.cfs", "_2.cfs"); String[] filesPre = dir.listAll(); Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -39,6 +39,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.SetBasedFieldSelector; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexReader.FieldOption; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.IndexSearcher; @@ -474,6 +475,7 @@ // add 11 documents with term : aaa writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); + writer.commit(); for (int i = 0; i < 11; i++) { addDoc(writer, searchTerm.text()); @@ -1765,6 +1767,7 @@ public void testPrepareCommitIsCurrent() throws Throwable { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); + writer.commit(); Document doc = new Document(); writer.addDocument(doc); IndexReader r = IndexReader.open(dir, true); Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -36,6 +36,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; @@ -172,6 +173,7 @@ private void doTestReopenWithCommit (Directory dir, boolean withReopen) throws IOException { IndexWriter iwriter = new IndexWriter(dir, new KeywordAnalyzer(), true, MaxFieldLength.LIMITED); iwriter.setMergeScheduler(new SerialMergeScheduler()); + iwriter.commit(); IndexReader reader = IndexReader.open(dir, false); try { int M = 3; Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -47,6 +47,7 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; @@ -783,7 +784,7 @@ writer.close(); long gen = SegmentInfos.getCurrentSegmentGeneration(dir); - assertTrue("segment generation should be > 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); // Make the next segments file, with last byte // missing, to simulate a writer that crashed while @@ -843,7 +844,7 @@ writer.close(); long gen = SegmentInfos.getCurrentSegmentGeneration(dir); - assertTrue("segment generation should be > 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir); String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, @@ -908,7 +909,7 @@ writer.close(); long gen = SegmentInfos.getCurrentSegmentGeneration(dir); - assertTrue("segment generation should be > 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); String[] files = dir.listAll(); for(int i=0;i 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir); IndexInput in = dir.openInput(segmentsFileName); @@ -2675,7 +2676,8 @@ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer.setMaxBufferedDocs(2); writer.setMergeFactor(5); - + writer.commit(); + for (int i = 0; i < 23; i++) addDoc(writer); @@ -3542,7 +3544,8 @@ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); writer.setMaxBufferedDocs(2); writer.setMergeFactor(5); - + writer.commit(); + for (int i = 0; i < 23; i++) addDoc(writer); @@ -3595,7 +3598,8 @@ writer.setMaxBufferedDocs(2); writer.setMergeFactor(5); - + writer.commit(); + for (int i = 0; i < 23; i++) addDoc(writer); @@ -3679,6 +3683,7 @@ dir2 = new MockRAMDirectory(); writer2 = new IndexWriter(dir2, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); + writer2.commit(); cms = (ConcurrentMergeScheduler) writer2.getMergeScheduler(); readers = new IndexReader[NUM_COPY]; Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -764,7 +765,7 @@ MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); - + modifier.commit(); dir.failOn(failure.reset()); for (int i = 0; i < keywords.length; i++) { Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -138,7 +138,8 @@ ((ConcurrentMergeScheduler) writer.getMergeScheduler()).setSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.setRAMBufferSizeMB(0.1); - + writer.commit(); + if (DEBUG) writer.setInfoStream(System.out); @@ -176,6 +177,7 @@ ((ConcurrentMergeScheduler) writer.getMergeScheduler()).setSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.setRAMBufferSizeMB(0.2); + writer.commit(); if (DEBUG) writer.setInfoStream(System.out); Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -642,6 +643,7 @@ Directory dir1 = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); + writer.commit(); writer.setInfoStream(infoStream); // create the index Index: lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.util.*; import org.apache.lucene.store.*; import org.apache.lucene.document.*; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.analysis.*; import org.apache.lucene.search.*; import org.apache.lucene.queryParser.*; @@ -121,7 +122,7 @@ */ public void runStressTest(Directory directory, MergeScheduler mergeScheduler) throws Exception { IndexWriter modifier = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); - + modifier.commit(); modifier.setMaxBufferedDocs(10); TimedThread[] threads = new TimedThread[4]; Index: lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java (working copy) @@ -16,6 +16,7 @@ import org.apache.lucene.store.*; import org.apache.lucene.document.*; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.analysis.*; import org.apache.lucene.util.LuceneTestCase; @@ -124,6 +125,7 @@ public DocsAndWriter indexRandomIWReader(int nThreads, int iterations, int range, Directory dir) throws IOException, InterruptedException { Map docs = new HashMap(); IndexWriter w = new MockIndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); + w.commit(); w.setUseCompoundFile(false); /*** Index: lucene/backwards/src/test/org/apache/lucene/store/TestBufferedIndexInput.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (working copy) @@ -241,6 +241,7 @@ public void testSetBufferSize() throws IOException { File indexDir = new File(System.getProperty("tempDir"), "testSetBufferSize"); + indexDir.mkdirs(); // required for this MockFSDir since we don't commit on IW creation anymore. MockFSDirectory dir = new MockFSDirectory(indexDir, newRandom()); try { IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); Index: lucene/backwards/src/test/org/apache/lucene/store/TestLockFactory.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/store/TestLockFactory.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/store/TestLockFactory.java (working copy) @@ -85,7 +85,8 @@ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); - + writer.commit(); // required so the second open succeed + // Create a 2nd IndexWriter. This is normally not allowed but it should run through since we're not // using any locks: IndexWriter writer2 = null; Index: lucene/backwards/src/test/org/apache/lucene/store/TestWindowsMMap.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/store/TestWindowsMMap.java (revision 932917) +++ lucene/backwards/src/test/org/apache/lucene/store/TestWindowsMMap.java (working copy) @@ -64,14 +64,18 @@ new File(System.getProperty("tempDir"),"testLuceneMmap").getAbsolutePath(); public void testMmapIndex() throws Exception { - FSDirectory storeDirectory; - storeDirectory = new MMapDirectory(new File(storePathname), null); + // sometimes the directory is not cleaned by rmDir, because on Windows it + // may take some time until the files are finally dereferenced. So clean the + // directory up front, or otherwise new IndexWriter will fail. + rmDir(new File(storePathname)); + FSDirectory storeDirectory = new MMapDirectory(new File(storePathname), null); // plan to add a set of useful stopwords, consider changing some of the // interior filters. StandardAnalyzer analyzer = new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT, Collections.emptySet()); // TODO: something about lock timeouts and leftover locks. IndexWriter writer = new IndexWriter(storeDirectory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); + writer.commit(); IndexSearcher searcher = new IndexSearcher(storeDirectory, true); for(int dx = 0; dx < 1000; dx ++) { @@ -83,14 +87,16 @@ searcher.close(); writer.close(); - rmDir(new File(storePathname)); + rmDir(new File(storePathname)); } - private void rmDir(File dir) { - File[] files = dir.listFiles(); - for (int i = 0; i < files.length; i++) { - files[i].delete(); - } - dir.delete(); - } + private void rmDir(File dir) { + if (!dir.exists()) { + return; + } + for (File file : dir.listFiles()) { + file.delete(); + } + dir.delete(); + } } Index: lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java (revision 932917) +++ lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.search.Filter; import org.apache.lucene.search.NumericRangeFilter; @@ -62,7 +63,8 @@ Filter filter = filterBuilder.getFilter(doc.getDocumentElement()); RAMDirectory ramDir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(ramDir, null, MaxFieldLength.UNLIMITED); + IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)); + writer.commit(); try { IndexReader reader = IndexReader.open(ramDir, true); Index: lucene/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DirectoryReader.java (revision 932917) +++ lucene/src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -1030,7 +1030,11 @@ Collection commits = new ArrayList(); SegmentInfos latest = new SegmentInfos(); - latest.read(dir, codecs); + try { + latest.read(dir, codecs); + } catch (IndexNotFoundException e) { + return Collections.emptyList(); + } final long currentGen = latest.getGeneration(); commits.add(new ReaderCommit(latest, dir)); Index: lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 932917) +++ lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NoSuchDirectoryException; /* * This class keeps track of each SegmentInfos instance that @@ -144,58 +145,62 @@ long currentGen = segmentInfos.getGeneration(); indexFilenameFilter = new IndexFileNameFilter(codecs); - String[] files = directory.listAll(); - CommitPoint currentCommitPoint = null; + boolean seenIndexFiles = false; + try { + for (String fileName : directory.listAll()) { - for(int i=0;i commits = IndexReader.listCommits(dir); - // 1 from opening writer + 2 from closing writer - assertEquals(3, commits.size()); + // 2 from closing writer + assertEquals(2, commits.size()); // Make sure we can open a reader on each commit: for (final IndexCommit commit : commits) { @@ -374,7 +374,7 @@ writer.close(); Collection commits = IndexReader.listCommits(dir); - assertEquals(6, commits.size()); + assertEquals(5, commits.size()); IndexCommit lastCommit = null; for (final IndexCommit commit : commits) { if (lastCommit == null || commit.getGeneration() > lastCommit.getGeneration()) @@ -389,7 +389,7 @@ writer.optimize(); writer.close(); - assertEquals(7, IndexReader.listCommits(dir).size()); + assertEquals(6, IndexReader.listCommits(dir).size()); // Now open writer on the commit just before optimize: writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) @@ -412,7 +412,7 @@ writer.close(); // Now 8 because we made another commit - assertEquals(8, IndexReader.listCommits(dir).size()); + assertEquals(7, IndexReader.listCommits(dir).size()); r = IndexReader.open(dir, true); // Not optimized because we rolled it back, and now only @@ -491,7 +491,7 @@ writer.optimize(); writer.close(); - assertEquals(2, policy.numOnInit); + assertEquals(1, policy.numOnInit); // If we are not auto committing then there should // be exactly 2 commits (one per close above): assertEquals(2, policy.numOnCommit); @@ -537,7 +537,7 @@ } assertTrue(policy.numDelete > 0); - assertEquals(N+1, policy.numOnInit); + assertEquals(N, policy.numOnInit); assertEquals(N+1, policy.numOnCommit); // Simplistic check: just verify only the past N segments_N's still @@ -625,8 +625,8 @@ // this is a commit writer.close(); - assertEquals(2*(N+2), policy.numOnInit); - assertEquals(2*(N+2)-1, policy.numOnCommit); + assertEquals(2*(N+1)+1, policy.numOnInit); + assertEquals(2*(N+2), policy.numOnCommit); IndexSearcher searcher = new IndexSearcher(dir, false); ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; @@ -735,8 +735,8 @@ writer.close(); } - assertEquals(1+3*(N+1), policy.numOnInit); - assertEquals(3*(N+1), policy.numOnCommit); + assertEquals(3*(N+1), policy.numOnInit); + assertEquals(3*(N+1)+1, policy.numOnCommit); IndexSearcher searcher = new IndexSearcher(dir, false); ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; Index: lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (working copy) @@ -138,11 +138,11 @@ copyFile(dir, "_0.cfs", "deletable"); // Create some old segments file: - copyFile(dir, "segments_3", "segments"); - copyFile(dir, "segments_3", "segments_2"); + copyFile(dir, "segments_2", "segments"); + copyFile(dir, "segments_2", "segments_1"); // Create a bogus cfs file shadowing a non-cfs segment: - copyFile(dir, "_2.cfs", "_3.cfs"); + copyFile(dir, "_1.cfs", "_2.cfs"); String[] filesPre = dir.listAll(); Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -466,18 +466,17 @@ public void testLockObtainFailed() throws IOException { Directory dir = new RAMDirectory(); - IndexWriter writer = null; - IndexReader reader = null; Term searchTerm = new Term("content", "aaa"); // add 11 documents with term : aaa - writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + writer.commit(); for (int i = 0; i < 11; i++) { addDoc(writer, searchTerm.text()); } // Create reader: - reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir, false); // Try to make changes try { @@ -1749,6 +1748,7 @@ Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + writer.commit(); Document doc = new Document(); writer.addDocument(doc); IndexReader r = IndexReader.open(dir, true); Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -174,6 +174,7 @@ IndexWriter iwriter = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new KeywordAnalyzer()).setOpenMode( OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler())); + iwriter.commit(); IndexReader reader = IndexReader.open(dir, false); try { int M = 3; Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -70,6 +70,7 @@ import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.MockRAMDirectory; +import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.UnicodeUtil; @@ -778,7 +779,7 @@ writer.close(); long gen = SegmentInfos.getCurrentSegmentGeneration(dir); - assertTrue("segment generation should be > 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); // Make the next segments file, with last byte // missing, to simulate a writer that crashed while @@ -838,7 +839,7 @@ writer.close(); long gen = SegmentInfos.getCurrentSegmentGeneration(dir); - assertTrue("segment generation should be > 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir); String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, @@ -903,7 +904,7 @@ writer.close(); long gen = SegmentInfos.getCurrentSegmentGeneration(dir); - assertTrue("segment generation should be > 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); String[] files = dir.listAll(); for(int i=0;i 1 but got " + gen, gen > 1); + assertTrue("segment generation should be > 0 but got " + gen, gen > 0); final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir); IndexInput in = dir.openInput(segmentsFileName); @@ -2673,7 +2674,8 @@ TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) .setMaxBufferedDocs(2)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5); - + writer.commit(); + for (int i = 0; i < 23; i++) addDoc(writer); @@ -3534,7 +3536,8 @@ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(2)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5); - + writer.commit(); + for (int i = 0; i < 23; i++) addDoc(writer); @@ -3585,7 +3588,8 @@ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(2)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5); - + writer.commit(); + for (int i = 0; i < 23; i++) addDoc(writer); @@ -3670,6 +3674,7 @@ dir2 = new MockRAMDirectory(); writer2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + writer2.commit(); cms = (ConcurrentMergeScheduler) writer2.getConfig().getMergeScheduler(); readers = new IndexReader[NUM_COPY]; @@ -4952,4 +4957,25 @@ w.close(); dir.close(); } + + public void testNoCommits() throws Exception { + // Tests that if we don't call commit(), the directory has 0 commits. This has + // changed since LUCENE-2386, where before IW would always commit on a fresh + // new index. + Directory dir = new RAMDirectory(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + assertEquals("expected 0 commits!", 0, IndexReader.listCommits(dir).size()); + // No changes still should generate a commit, because it's a new index. + writer.close(); + assertEquals("expected 1 commits!", 1, IndexReader.listCommits(dir).size()); + } + + public void testEmptyFSDirWithNoLock() throws Exception { + // Tests that if FSDir is opened w/ a NoLockFactory (or SingleInstanceLF), + // then IndexWriter ctor succeeds. Previously (LUCENE-2386) it failed + // when listAll() was called in IndexFileDeleter. + FSDirectory dir = FSDirectory.open(new File(TEMP_DIR, "emptyFSDirNoLock"), NoLockFactory.getNoLockFactory()); + new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))).close(); + } + } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy) @@ -749,7 +749,7 @@ MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - + modifier.commit(); dir.failOn(failure.reset()); for (int i = 0; i < keywords.length; i++) { Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -134,6 +134,7 @@ MockIndexWriter writer = new MockIndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setRAMBufferSizeMB(0.1)); ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions(); //writer.setMaxBufferedDocs(10); + writer.commit(); if (VERBOSE) writer.setInfoStream(System.out); @@ -171,6 +172,7 @@ MockIndexWriter writer = new MockIndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setRAMBufferSizeMB(0.2)); ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions(); //writer.setMaxBufferedDocs(10); + writer.commit(); if (VERBOSE) writer.setInfoStream(System.out); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (working copy) @@ -561,6 +561,7 @@ public void testAfterCommit() throws Exception { Directory dir1 = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir1, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + writer.commit(); writer.setInfoStream(infoStream); // create the index Index: lucene/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java (working copy) @@ -82,10 +82,7 @@ doc.add(new Field("c", "a" + i, Store.YES, Index.ANALYZED)); writer.addDocument(doc); writer.commit(); - // the reason to expect i + 2 commits is because when IndexWriter is - // created it creates a first commit. If this ever changes, then the - // expected should be i + 1 (and this comment removed). - assertEquals("wrong number of commits !", i + 2, IndexReader.listCommits(dir).size()); + assertEquals("wrong number of commits !", i + 1, IndexReader.listCommits(dir).size()); } writer.close(); } Index: lucene/src/test/org/apache/lucene/index/TestStressIndexing.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestStressIndexing.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestStressIndexing.java (working copy) @@ -122,7 +122,8 @@ TEST_VERSION_CURRENT, new SimpleAnalyzer(TEST_VERSION_CURRENT)) .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(10).setMergeScheduler( mergeScheduler)); - + modifier.commit(); + TimedThread[] threads = new TimedThread[4]; int numThread = 0; Index: lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 932917) +++ lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (working copy) @@ -150,6 +150,7 @@ IndexWriter w = new MockIndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setRAMBufferSizeMB( 0.1).setMaxBufferedDocs(maxBufferedDocs)); + w.commit(); LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); lmp.setUseCompoundDocStore(false); Index: lucene/src/test/org/apache/lucene/store/TestLockFactory.java =================================================================== --- lucene/src/test/org/apache/lucene/store/TestLockFactory.java (revision 932917) +++ lucene/src/test/org/apache/lucene/store/TestLockFactory.java (working copy) @@ -82,7 +82,7 @@ NoLockFactory.class.isInstance(dir.getLockFactory())); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - + writer.commit(); // required so the second open succeed // Create a 2nd IndexWriter. This is normally not allowed but it should run through since we're not // using any locks: IndexWriter writer2 = null; Index: lucene/src/test/org/apache/lucene/store/TestWindowsMMap.java =================================================================== --- lucene/src/test/org/apache/lucene/store/TestWindowsMMap.java (revision 932917) +++ lucene/src/test/org/apache/lucene/store/TestWindowsMMap.java (working copy) @@ -66,17 +66,22 @@ new File(TEMP_DIR,"testLuceneMmap").getAbsolutePath(); public void testMmapIndex() throws Exception { - FSDirectory storeDirectory; - storeDirectory = new MMapDirectory(new File(storePathname), null); - + // sometimes the directory is not cleaned by rmDir, because on Windows it + // may take some time until the files are finally dereferenced. So clean the + // directory up front, or otherwise new IndexWriter will fail. + File dirPath = new File(storePathname); + rmDir(dirPath); + MMapDirectory dir = new MMapDirectory(dirPath, null); + // plan to add a set of useful stopwords, consider changing some of the // interior filters. StandardAnalyzer analyzer = new StandardAnalyzer(TEST_VERSION_CURRENT, Collections.emptySet()); // TODO: something about lock timeouts and leftover locks. - IndexWriter writer = new IndexWriter(storeDirectory, new IndexWriterConfig( + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, analyzer) .setOpenMode(OpenMode.CREATE)); - IndexSearcher searcher = new IndexSearcher(storeDirectory, true); + writer.commit(); + IndexSearcher searcher = new IndexSearcher(dir, true); for(int dx = 0; dx < 1000; dx ++) { String f = randomField(); @@ -87,14 +92,16 @@ searcher.close(); writer.close(); - rmDir(new File(storePathname)); + rmDir(dirPath); } - private void rmDir(File dir) { - File[] files = dir.listFiles(); - for (int i = 0; i < files.length; i++) { - files[i].delete(); - } - dir.delete(); - } + private void rmDir(File dir) { + if (!dir.exists()) { + return; + } + for (File file : dir.listFiles()) { + file.delete(); + } + dir.delete(); + } }