Index: lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java (working copy) @@ -24,6 +24,8 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; +import org.junit.AfterClass; +import org.junit.BeforeClass; import java.text.DecimalFormat; import java.util.Random; @@ -32,14 +34,12 @@ */ public class TestBooleanMinShouldMatch extends LuceneTestCase { - private Directory index; - private IndexReader r; - private IndexSearcher s; + private static Directory index; + private static IndexReader r; + private static IndexSearcher s; - @Override - public void setUp() throws Exception { - super.setUp(); - + @BeforeClass + public static void beforeClass() throws Exception { String[] data = new String [] { "A 1 2 3 4 5 6", "Z 4 5 6", @@ -70,12 +70,14 @@ //System.out.println("Set up " + getName()); } - @Override - public void tearDown() throws Exception { + @AfterClass + public static void afterClass() throws Exception { s.close(); + s = null; r.close(); + r = null; index.close(); - super.tearDown(); + index = null; } @@ -312,7 +314,7 @@ // increase number of iterations for more complete testing - int num = 50 * RANDOM_MULTIPLIER; + int num = (TEST_NIGHTLY ? 50 : 10) * RANDOM_MULTIPLIER; for (int i=0; i 0 && buffer[offset] >= 0xdc00 && buffer[offset] < 0xe000) + // Don't start in the middle of a valid surrogate pair + offset--; + + for(int i=offset;i allTerms, boolean isTop) throws IOException { + TermsEnum terms = MultiFields.getFields(r).terms("f").iterator(); + + BytesRef last = new BytesRef(); + + Set seenTerms = new HashSet(); + + while(true) { + final BytesRef term = terms.next(); + if (term == null) { + break; + } + + assertTrue(last.compareTo(term) < 0); + last.copy(term); + + final String s = term.utf8ToString(); + assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s)); + seenTerms.add(s); + } + + if (isTop) { + assertTrue(allTerms.equals(seenTerms)); + } + + // Test seeking: + Iterator it = seenTerms.iterator(); + while(it.hasNext()) { + BytesRef tr = new BytesRef(it.next()); + assertEquals("seek failed for term=" + termDesc(tr.utf8ToString()), + TermsEnum.SeekStatus.FOUND, + terms.seek(tr)); + } + } + + // LUCENE-510 + public void testRandomUnicodeStrings() throws Throwable { + char[] buffer = new char[20]; + char[] expected = new char[20]; + + BytesRef utf8 = new BytesRef(20); + CharsRef utf16 = new CharsRef(20); + + int num = 100000 * RANDOM_MULTIPLIER; + for (int iter = 0; iter < num; iter++) { + boolean hasIllegal = fillUnicode(buffer, expected, 0, 20); + + UnicodeUtil.UTF16toUTF8(buffer, 0, 20, utf8); + if (!hasIllegal) { + byte[] b = new String(buffer, 0, 20).getBytes("UTF-8"); + assertEquals(b.length, utf8.length); + for(int i=0;i> 10) + UnicodeUtil.UNI_SUR_HIGH_START); + chars[len++] = (char) (((ch-0x0010000) & 0x3FFL) + UnicodeUtil.UNI_SUR_LOW_START); + } + + UnicodeUtil.UTF16toUTF8(chars, 0, len, utf8); + + String s1 = new String(chars, 0, len); + String s2 = new String(utf8.bytes, 0, utf8.length, "UTF-8"); + assertEquals("codepoint " + ch, s1, s2); + + UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16); + assertEquals("codepoint " + ch, s1, new String(utf16.chars, 0, utf16.length)); + + byte[] b = s1.getBytes("UTF-8"); + assertEquals(utf8.length, b.length); + for(int j=0;j allTerms = new HashSet(); + + int num = 200 * RANDOM_MULTIPLIER; + for (int i = 0; i < num; i++) { + + final String s; + if (rnd.nextBoolean()) { + // Single char + if (rnd.nextBoolean()) { + // Above surrogates + chars[0] = (char) getInt(rnd, 1+UnicodeUtil.UNI_SUR_LOW_END, 0xffff); + } else { + // Below surrogates + chars[0] = (char) getInt(rnd, 0, UnicodeUtil.UNI_SUR_HIGH_START-1); + } + s = new String(chars, 0, 1); + } else { + // Surrogate pair + chars[0] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_HIGH_START, UnicodeUtil.UNI_SUR_HIGH_END); + assertTrue(((int) chars[0]) >= UnicodeUtil.UNI_SUR_HIGH_START && ((int) chars[0]) <= UnicodeUtil.UNI_SUR_HIGH_END); + chars[1] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_LOW_START, UnicodeUtil.UNI_SUR_LOW_END); + s = new String(chars, 0, 2); + } + allTerms.add(s); + f.setValue(s); + + writer.addDocument(d); + + if ((1+i) % 42 == 0) { + writer.commit(); + } + } + + IndexReader r = writer.getReader(); + + // Test each sub-segment + final IndexReader[] subs = r.getSequentialSubReaders(); + for(int i=0;i commitData = new HashMap(); + commitData.put("tag", "first"); + w.commit(commitData); + + // commit to "second" + w.addDocument(doc); + commitData.put("tag", "second"); + w.commit(commitData); + w.close(); + + // open "first" with IndexWriter + IndexCommit commit = null; + for(IndexCommit c : IndexReader.listCommits(dir)) { + if (c.getUserData().get("tag").equals("first")) { + commit = c; + break; + } + } + + assertNotNull(commit); + + w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit)); + + assertEquals(1, w.numDocs()); + + // commit IndexWriter to "third" + w.addDocument(doc); + commitData.put("tag", "third"); + w.commit(commitData); + w.close(); + + // make sure "second" commit is still there + commit = null; + for(IndexCommit c : IndexReader.listCommits(dir)) { + if (c.getUserData().get("tag").equals("second")) { + commit = c; + break; + } + } + + assertNotNull(commit); + + IndexReader r = IndexReader.open(commit, true); + assertEquals(2, r.numDocs()); + r.close(); + + // open "second", w/ writeable IndexReader & commit + r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false); + assertEquals(2, r.numDocs()); + r.deleteDocument(0); + r.deleteDocument(1); + commitData.put("tag", "fourth"); + r.commit(commitData); + r.close(); + + // make sure "third" commit is still there + commit = null; + for(IndexCommit c : IndexReader.listCommits(dir)) { + if (c.getUserData().get("tag").equals("third")) { + commit = c; + break; + } + } + assertNotNull(commit); + + dir.close(); + } + + public void testNoCommits() throws Exception { + // Tests that if we don't call commit(), the directory has 0 commits. This has + // changed since LUCENE-2386, where before IW would always commit on a fresh + // new index. + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); + try { + IndexReader.listCommits(dir); + fail("listCommits should have thrown an exception over empty index"); + } catch (IndexNotFoundException e) { + // that's expected ! + } + // No changes still should generate a commit, because it's a new index. + writer.close(); + assertEquals("expected 1 commits!", 1, IndexReader.listCommits(dir).size()); + dir.close(); + } + + // LUCENE-1274: test writer.prepareCommit() + public void testPrepareCommit() throws IOException { + Directory dir = newDirectory(); + + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDocs(2). + setMergePolicy(newLogMergePolicy(5)) + ); + writer.commit(); + + for (int i = 0; i < 23; i++) + TestIndexWriter.addDoc(writer); + + IndexReader reader = IndexReader.open(dir, true); + assertEquals(0, reader.numDocs()); + + writer.prepareCommit(); + + IndexReader reader2 = IndexReader.open(dir, true); + assertEquals(0, reader2.numDocs()); + + writer.commit(); + + IndexReader reader3 = reader.reopen(); + assertEquals(0, reader.numDocs()); + assertEquals(0, reader2.numDocs()); + assertEquals(23, reader3.numDocs()); + reader.close(); + reader2.close(); + + for (int i = 0; i < 17; i++) + TestIndexWriter.addDoc(writer); + + assertEquals(23, reader3.numDocs()); + reader3.close(); + reader = IndexReader.open(dir, true); + assertEquals(23, reader.numDocs()); + reader.close(); + + writer.prepareCommit(); + + reader = IndexReader.open(dir, true); + assertEquals(23, reader.numDocs()); + reader.close(); + + writer.commit(); + reader = IndexReader.open(dir, true); + assertEquals(40, reader.numDocs()); + reader.close(); + writer.close(); + dir.close(); + } + + // LUCENE-1274: test writer.prepareCommit() + public void testPrepareCommitRollback() throws IOException { + MockDirectoryWrapper dir = newDirectory(); + dir.setPreventDoubleWrite(false); + + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDocs(2). + setMergePolicy(newLogMergePolicy(5)) + ); + writer.commit(); + + for (int i = 0; i < 23; i++) + TestIndexWriter.addDoc(writer); + + IndexReader reader = IndexReader.open(dir, true); + assertEquals(0, reader.numDocs()); + + writer.prepareCommit(); + + IndexReader reader2 = IndexReader.open(dir, true); + assertEquals(0, reader2.numDocs()); + + writer.rollback(); + + IndexReader reader3 = reader.reopen(); + assertEquals(0, reader.numDocs()); + assertEquals(0, reader2.numDocs()); + assertEquals(0, reader3.numDocs()); + reader.close(); + reader2.close(); + + writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); + for (int i = 0; i < 17; i++) + TestIndexWriter.addDoc(writer); + + assertEquals(0, reader3.numDocs()); + reader3.close(); + reader = IndexReader.open(dir, true); + assertEquals(0, reader.numDocs()); + reader.close(); + + writer.prepareCommit(); + + reader = IndexReader.open(dir, true); + assertEquals(0, reader.numDocs()); + reader.close(); + + writer.commit(); + reader = IndexReader.open(dir, true); + assertEquals(17, reader.numDocs()); + reader.close(); + writer.close(); + dir.close(); + } + + // LUCENE-1274 + public void testPrepareCommitNoChanges() throws IOException { + Directory dir = newDirectory(); + + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); + writer.prepareCommit(); + writer.commit(); + writer.close(); + + IndexReader reader = IndexReader.open(dir, true); + assertEquals(0, reader.numDocs()); + reader.close(); + dir.close(); + } + + // LUCENE-1382 + public void testCommitUserData() throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2)); + for(int j=0;j<17;j++) + TestIndexWriter.addDoc(w); + w.close(); + + assertEquals(0, IndexReader.getCommitUserData(dir).size()); + + IndexReader r = IndexReader.open(dir, true); + // commit(Map) never called for this index + assertEquals(0, r.getCommitUserData().size()); + r.close(); + + w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2)); + for(int j=0;j<17;j++) + TestIndexWriter.addDoc(w); + Map data = new HashMap(); + data.put("label", "test1"); + w.commit(data); + w.close(); + + assertEquals("test1", IndexReader.getCommitUserData(dir).get("label")); + + r = IndexReader.open(dir, true); + assertEquals("test1", r.getCommitUserData().get("label")); + r.close(); + + w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.optimize(); + w.close(); + + assertEquals("test1", IndexReader.getCommitUserData(dir).get("label")); + + dir.close(); + } +} Index: lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java (working copy) @@ -51,13 +51,13 @@ Document doc = new Document(); doc.add(newField(fieldName, "1 2 3 4 5 6 7 8 9 10 " + "1 2 3 4 5 6 7 8 9 10 " + "1 2 3 4 5 6 7 8 9 10 " - + "1 2 3 4 5 6 7 8 9 10", Field.Store.YES, Field.Index.ANALYZED)); + + "1 2 3 4 5 6 7 8 9 10", Field.Store.NO, Field.Index.ANALYZED_NO_NORMS)); writer.addDocument(doc); } IndexReader reader = writer.getReader(); writer.close(); - for (int i = 0; i < 39 * RANDOM_MULTIPLIER; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13) * RANDOM_MULTIPLIER; i++) { BytesRef bytes = new BytesRef("1"); ReaderContext topReaderContext = reader.getTopReaderContext(); AtomicReaderContext[] leaves = ReaderUtil.leaves(topReaderContext); @@ -112,7 +112,7 @@ Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - int numDocs = 131; + int numDocs = TEST_NIGHTLY ? 131 : 47; int max = 1051; int term = random.nextInt(max); Integer[][] positionsInDoc = new Integer[numDocs][]; @@ -120,7 +120,8 @@ Document doc = new Document(); ArrayList positions = new ArrayList(); StringBuilder builder = new StringBuilder(); - for (int j = 0; j < 3049; j++) { + int num = TEST_NIGHTLY ? 3049 : 499; + for (int j = 0; j < num; j++) { int nextInt = random.nextInt(max); builder.append(nextInt).append(" "); if (nextInt == term) { @@ -129,10 +130,10 @@ } if (positions.size() == 0) { builder.append(term); - positions.add(3049); + positions.add(num); } - doc.add(newField(fieldName, builder.toString(), Field.Store.YES, - Field.Index.ANALYZED)); + doc.add(newField(fieldName, builder.toString(), Field.Store.NO, + Field.Index.ANALYZED_NO_NORMS)); positionsInDoc[i] = positions.toArray(new Integer[0]); writer.addDocument(doc); } @@ -140,7 +141,7 @@ IndexReader reader = writer.getReader(); writer.close(); - for (int i = 0; i < 39 * RANDOM_MULTIPLIER; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13) * RANDOM_MULTIPLIER; i++) { BytesRef bytes = new BytesRef("" + term); ReaderContext topReaderContext = reader.getTopReaderContext(); AtomicReaderContext[] leaves = ReaderUtil.leaves(topReaderContext); @@ -192,7 +193,7 @@ Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - int numDocs = 499; + int numDocs = TEST_NIGHTLY ? 499 : 131; int max = 15678; int term = random.nextInt(max); int[] freqInDoc = new int[numDocs]; @@ -206,15 +207,15 @@ freqInDoc[i]++; } } - doc.add(newField(fieldName, builder.toString(), Field.Store.YES, - Field.Index.ANALYZED)); + doc.add(newField(fieldName, builder.toString(), Field.Store.NO, + Field.Index.ANALYZED_NO_NORMS)); writer.addDocument(doc); } IndexReader reader = writer.getReader(); writer.close(); - for (int i = 0; i < 39 * RANDOM_MULTIPLIER; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13) * RANDOM_MULTIPLIER; i++) { BytesRef bytes = new BytesRef("" + term); ReaderContext topReaderContext = reader.getTopReaderContext(); AtomicReaderContext[] leaves = ReaderUtil.leaves(topReaderContext); @@ -281,8 +282,8 @@ builder.append("odd "); } } - doc.add(newField(fieldName, builder.toString(), Field.Store.YES, - Field.Index.ANALYZED)); + doc.add(newField(fieldName, builder.toString(), Field.Store.NO, + Field.Index.ANALYZED_NO_NORMS)); writer.addDocument(doc); } @@ -290,7 +291,7 @@ IndexReader reader = writer.getReader(); writer.close(); - for (int i = 0; i < 39 * RANDOM_MULTIPLIER; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13) * RANDOM_MULTIPLIER; i++) { BytesRef bytes = new BytesRef("even"); ReaderContext topReaderContext = reader.getTopReaderContext(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterOptimize.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterOptimize.java (revision 0) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterOptimize.java (revision 0) @@ -0,0 +1,215 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +public class TestIndexWriterOptimize extends LuceneTestCase { + public void testOptimizeMaxNumSegments() throws IOException { + + MockDirectoryWrapper dir = newDirectory(); + + final Document doc = new Document(); + doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED)); + final int incrMin = TEST_NIGHTLY ? 15 : 40; + for(int numDocs=10;numDocs<500;numDocs += _TestUtil.nextInt(random, incrMin, 5*incrMin)) { + LogDocMergePolicy ldmp = new LogDocMergePolicy(); + ldmp.setMinMergeDocs(1); + ldmp.setMergeFactor(5); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(2).setMergePolicy( + ldmp)); + for(int j=0;j= 2.0) { + rate /= 2; + } + if (diskRatio >= 4.0) { + rate /= 2; + } + if (diskRatio >= 6.0) { + rate = 0.0; + } + if (VERBOSE) { + System.out.println("\ncycle: " + diskFree + " bytes"); + } + testName = "disk full during reader.close() @ " + thisDiskFree + " bytes"; + } else { + thisDiskFree = 0; + rate = 0.0; + if (VERBOSE) { + System.out.println("\ncycle: same writer: unlimited disk space"); + } + testName = "reader re-use after disk full"; + } + + dir.setMaxSizeInBytes(thisDiskFree); + dir.setRandomIOExceptionRate(rate); + Similarity sim = new DefaultSimilarity(); + try { + if (0 == x) { + int docId = 12; + for(int i=0;i<13;i++) { + reader.deleteDocument(docId); + reader.setNorm(docId, "content", sim.encodeNormValue(2.0f)); + docId += 12; + } + } + reader.close(); + success = true; + if (0 == x) { + done = true; + } + } catch (IOException e) { + if (VERBOSE) { + System.out.println(" hit IOException: " + e); + e.printStackTrace(System.out); + } + err = e; + gotExc = true; + if (1 == x) { + e.printStackTrace(); + fail(testName + " hit IOException after disk space was freed up"); + } + } + + // Finally, verify index is not corrupt, and, if + // we succeeded, we see all docs changed, and if + // we failed, we see either all docs or no docs + // changed (transactional semantics): + IndexReader newReader = null; + try { + newReader = IndexReader.open(dir, false); + } catch (IOException e) { + e.printStackTrace(); + fail(testName + ":exception when creating IndexReader after disk full during close: " + e); + } + /* + int result = newReader.docFreq(searchTerm); + if (success) { + if (result != END_COUNT) { + fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + END_COUNT); + } + } else { + // On hitting exception we still may have added + // all docs: + if (result != START_COUNT && result != END_COUNT) { + err.printStackTrace(); + fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT); + } + } + */ + + IndexSearcher searcher = newSearcher(newReader); + ScoreDoc[] hits = null; + try { + hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; + } catch (IOException e) { + e.printStackTrace(); + fail(testName + ": exception when searching: " + e); + } + int result2 = hits.length; + if (success) { + if (result2 != END_COUNT) { + fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT); + } + } else { + // On hitting exception we still may have added + // all docs: + if (result2 != START_COUNT && result2 != END_COUNT) { + err.printStackTrace(); + fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT); + } + } + + searcher.close(); + newReader.close(); + + if (result2 == END_COUNT) { + if (!gotExc) + fail("never hit disk full"); + break; + } + } + + dir.close(); + + // Try again with 10 more bytes of free space: + diskFree += 10; + } + + startDir.close(); + } +} Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -304,7 +304,7 @@ d.close(); } - private void assertTermDocsCount(String msg, + static void assertTermDocsCount(String msg, IndexReader reader, Term term, int expected) @@ -322,50 +322,6 @@ assertEquals(msg + ", count mismatch", expected, count); } - public void testBasicDelete() throws IOException { - Directory dir = newDirectory(); - - IndexWriter writer = null; - IndexReader reader = null; - Term searchTerm = new Term("content", "aaa"); - - // add 100 documents with term : aaa - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - writer.setInfoStream(VERBOSE ? System.out : null); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm.text()); - } - writer.close(); - - // OPEN READER AT THIS POINT - this should fix the view of the - // index at the point of having 100 "aaa" documents and 0 "bbb" - reader = IndexReader.open(dir, false); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - reader.close(); - - // DELETE DOCUMENTS CONTAINING TERM: aaa - int deleted = 0; - reader = IndexReader.open(dir, false); - deleted = reader.deleteDocuments(searchTerm); - assertEquals("deleted count", 100, deleted); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - - // open a 2nd reader to make sure first reader can - // commit its changes (.del) while second reader - // is open: - IndexReader reader2 = IndexReader.open(dir, false); - reader.close(); - - // CREATE A NEW READER and re-test - reader = IndexReader.open(dir, false); - assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - reader.close(); - reader2.close(); - dir.close(); - } public void testBinaryFields() throws IOException { Directory dir = newDirectory(); @@ -600,11 +556,6 @@ dir.close(); } - - public void testDeleteReaderWriterConflictUnoptimized() throws IOException{ - deleteReaderWriterConflict(false); - } - /* ??? public void testOpenEmptyDirectory() throws IOException{ String dirName = "test.empty"; File fileDirName = new File(dirName); @@ -620,90 +571,6 @@ rmDir(fileDirName); }*/ - public void testDeleteReaderWriterConflictOptimized() throws IOException{ - deleteReaderWriterConflict(true); - } - - private void deleteReaderWriterConflict(boolean optimize) throws IOException { - //Directory dir = new RAMDirectory(); - Directory dir = newDirectory(); - - Term searchTerm = new Term("content", "aaa"); - Term searchTerm2 = new Term("content", "bbb"); - - // add 100 documents with term : aaa - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm.text()); - } - writer.close(); - - // OPEN READER AT THIS POINT - this should fix the view of the - // index at the point of having 100 "aaa" documents and 0 "bbb" - IndexReader reader = IndexReader.open(dir, false); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("first docFreq", 0, reader.docFreq(searchTerm2)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - assertTermDocsCount("first reader", reader, searchTerm2, 0); - - // add 100 documents with term : bbb - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm2.text()); - } - - // REQUEST OPTIMIZATION - // This causes a new segment to become current for all subsequent - // searchers. Because of this, deletions made via a previously open - // reader, which would be applied to that reader's segment, are lost - // for subsequent searchers/readers - if(optimize) - writer.optimize(); - writer.close(); - - // The reader should not see the new data - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("first docFreq", 0, reader.docFreq(searchTerm2)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - assertTermDocsCount("first reader", reader, searchTerm2, 0); - - - // DELETE DOCUMENTS CONTAINING TERM: aaa - // NOTE: the reader was created when only "aaa" documents were in - int deleted = 0; - try { - deleted = reader.deleteDocuments(searchTerm); - fail("Delete allowed on an index reader with stale segment information"); - } catch (StaleReaderException e) { - /* success */ - } - - // Re-open index reader and try again. This time it should see - // the new data. - reader.close(); - reader = IndexReader.open(dir, false); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm2)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - assertTermDocsCount("first reader", reader, searchTerm2, 100); - - deleted = reader.deleteDocuments(searchTerm); - assertEquals("deleted count", 100, deleted); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100); - reader.close(); - - // CREATE A NEW READER and re-test - reader = IndexReader.open(dir, false); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100); - reader.close(); - dir.close(); - } - public void testFilesOpenClose() throws IOException { // Create initial data set File dirFile = _TestUtil.getTempDir("TestIndexReader.testFilesOpenClose"); @@ -812,259 +679,6 @@ dir.close(); } - public void testUndeleteAll() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - addDocumentWithFields(writer); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(0); - reader.deleteDocument(1); - reader.undeleteAll(); - reader.close(); - reader = IndexReader.open(dir, false); - assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll() - reader.close(); - dir.close(); - } - - public void testUndeleteAllAfterClose() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - addDocumentWithFields(writer); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(0); - reader.close(); - reader = IndexReader.open(dir, false); - reader.undeleteAll(); - assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll() - reader.close(); - dir.close(); - } - - public void testUndeleteAllAfterCloseThenReopen() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - addDocumentWithFields(writer); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(0); - reader.close(); - reader = IndexReader.open(dir, false); - reader.undeleteAll(); - reader.close(); - reader = IndexReader.open(dir, false); - assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll() - reader.close(); - dir.close(); - } - - public void testDeleteReaderReaderConflictUnoptimized() throws IOException{ - deleteReaderReaderConflict(false); - } - - public void testDeleteReaderReaderConflictOptimized() throws IOException{ - deleteReaderReaderConflict(true); - } - - /** - * Make sure if reader tries to commit but hits disk - * full that reader remains consistent and usable. - */ - public void testDiskFull() throws IOException { - - Term searchTerm = new Term("content", "aaa"); - int START_COUNT = 157; - int END_COUNT = 144; - - // First build up a starting index: - MockDirectoryWrapper startDir = newDirectory(); - IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - if (VERBOSE) { - System.out.println("TEST: create initial index"); - writer.setInfoStream(System.out); - } - for(int i=0;i<157;i++) { - Document d = new Document(); - d.add(newField("id", Integer.toString(i), Field.Store.YES, Field.Index.NOT_ANALYZED)); - d.add(newField("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED)); - writer.addDocument(d); - if (0==i%10) - writer.commit(); - } - writer.close(); - - { - IndexReader r = IndexReader.open(startDir); - IndexSearcher searcher = newSearcher(r); - ScoreDoc[] hits = null; - try { - hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; - } catch (IOException e) { - e.printStackTrace(); - fail("exception when init searching: " + e); - } - searcher.close(); - r.close(); - } - - long diskUsage = startDir.getRecomputedActualSizeInBytes(); - long diskFree = diskUsage+100; - - IOException err = null; - - boolean done = false; - boolean gotExc = false; - - // Iterate w/ ever increasing free disk space: - while(!done) { - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir)); - - // If IndexReader hits disk full, it can write to - // the same files again. - dir.setPreventDoubleWrite(false); - - IndexReader reader = IndexReader.open(dir, false); - - // For each disk size, first try to commit against - // dir that will hit random IOExceptions & disk - // full; after, give it infinite disk space & turn - // off random IOExceptions & retry w/ same reader: - boolean success = false; - - for(int x=0;x<2;x++) { - - double rate = 0.05; - double diskRatio = ((double) diskFree)/diskUsage; - long thisDiskFree; - String testName; - - if (0 == x) { - thisDiskFree = diskFree; - if (diskRatio >= 2.0) { - rate /= 2; - } - if (diskRatio >= 4.0) { - rate /= 2; - } - if (diskRatio >= 6.0) { - rate = 0.0; - } - if (VERBOSE) { - System.out.println("\ncycle: " + diskFree + " bytes"); - } - testName = "disk full during reader.close() @ " + thisDiskFree + " bytes"; - } else { - thisDiskFree = 0; - rate = 0.0; - if (VERBOSE) { - System.out.println("\ncycle: same writer: unlimited disk space"); - } - testName = "reader re-use after disk full"; - } - - dir.setMaxSizeInBytes(thisDiskFree); - dir.setRandomIOExceptionRate(rate); - Similarity sim = new DefaultSimilarity(); - try { - if (0 == x) { - int docId = 12; - for(int i=0;i<13;i++) { - reader.deleteDocument(docId); - reader.setNorm(docId, "content", sim.encodeNormValue(2.0f)); - docId += 12; - } - } - reader.close(); - success = true; - if (0 == x) { - done = true; - } - } catch (IOException e) { - if (VERBOSE) { - System.out.println(" hit IOException: " + e); - e.printStackTrace(System.out); - } - err = e; - gotExc = true; - if (1 == x) { - e.printStackTrace(); - fail(testName + " hit IOException after disk space was freed up"); - } - } - - // Finally, verify index is not corrupt, and, if - // we succeeded, we see all docs changed, and if - // we failed, we see either all docs or no docs - // changed (transactional semantics): - IndexReader newReader = null; - try { - newReader = IndexReader.open(dir, false); - } catch (IOException e) { - e.printStackTrace(); - fail(testName + ":exception when creating IndexReader after disk full during close: " + e); - } - /* - int result = newReader.docFreq(searchTerm); - if (success) { - if (result != END_COUNT) { - fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + END_COUNT); - } - } else { - // On hitting exception we still may have added - // all docs: - if (result != START_COUNT && result != END_COUNT) { - err.printStackTrace(); - fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT); - } - } - */ - - IndexSearcher searcher = newSearcher(newReader); - ScoreDoc[] hits = null; - try { - hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; - } catch (IOException e) { - e.printStackTrace(); - fail(testName + ": exception when searching: " + e); - } - int result2 = hits.length; - if (success) { - if (result2 != END_COUNT) { - fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT); - } - } else { - // On hitting exception we still may have added - // all docs: - if (result2 != START_COUNT && result2 != END_COUNT) { - err.printStackTrace(); - fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT); - } - } - - searcher.close(); - newReader.close(); - - if (result2 == END_COUNT) { - if (!gotExc) - fail("never hit disk full"); - break; - } - } - - dir.close(); - - // Try again with 10 more bytes of free space: - diskFree += 10; - } - - startDir.close(); - } - public void testDocsOutOfOrderJIRA140() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); @@ -1161,133 +775,7 @@ dir.close(); } - public void testMultiReaderDeletes() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter w= new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - Document doc = new Document(); - doc.add(newField("f", "doctor", Field.Store.NO, Field.Index.NOT_ANALYZED)); - w.addDocument(doc); - doc = new Document(); - w.commit(); - doc.add(newField("f", "who", Field.Store.NO, Field.Index.NOT_ANALYZED)); - w.addDocument(doc); - IndexReader r = new SlowMultiReaderWrapper(w.getReader()); - w.close(); - - assertNull(r.getDeletedDocs()); - r.close(); - - r = new SlowMultiReaderWrapper(IndexReader.open(dir, false)); - - assertNull(r.getDeletedDocs()); - assertEquals(1, r.deleteDocuments(new Term("f", "doctor"))); - assertNotNull(r.getDeletedDocs()); - assertTrue(r.getDeletedDocs().get(0)); - assertEquals(1, r.deleteDocuments(new Term("f", "who"))); - assertTrue(r.getDeletedDocs().get(1)); - r.close(); - dir.close(); - } - - private void deleteReaderReaderConflict(boolean optimize) throws IOException { - Directory dir = newDirectory(); - - Term searchTerm1 = new Term("content", "aaa"); - Term searchTerm2 = new Term("content", "bbb"); - Term searchTerm3 = new Term("content", "ccc"); - - // add 100 documents with term : aaa - // add 100 documents with term : bbb - // add 100 documents with term : ccc - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm1.text()); - addDoc(writer, searchTerm2.text()); - addDoc(writer, searchTerm3.text()); - } - if(optimize) - writer.optimize(); - writer.close(); - - // OPEN TWO READERS - // Both readers get segment info as exists at this time - IndexReader reader1 = IndexReader.open(dir, false); - assertEquals("first opened", 100, reader1.docFreq(searchTerm1)); - assertEquals("first opened", 100, reader1.docFreq(searchTerm2)); - assertEquals("first opened", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("first opened", reader1, searchTerm1, 100); - assertTermDocsCount("first opened", reader1, searchTerm2, 100); - assertTermDocsCount("first opened", reader1, searchTerm3, 100); - - IndexReader reader2 = IndexReader.open(dir, false); - assertEquals("first opened", 100, reader2.docFreq(searchTerm1)); - assertEquals("first opened", 100, reader2.docFreq(searchTerm2)); - assertEquals("first opened", 100, reader2.docFreq(searchTerm3)); - assertTermDocsCount("first opened", reader2, searchTerm1, 100); - assertTermDocsCount("first opened", reader2, searchTerm2, 100); - assertTermDocsCount("first opened", reader2, searchTerm3, 100); - - // DELETE DOCS FROM READER 2 and CLOSE IT - // delete documents containing term: aaa - // when the reader is closed, the segment info is updated and - // the first reader is now stale - reader2.deleteDocuments(searchTerm1); - assertEquals("after delete 1", 100, reader2.docFreq(searchTerm1)); - assertEquals("after delete 1", 100, reader2.docFreq(searchTerm2)); - assertEquals("after delete 1", 100, reader2.docFreq(searchTerm3)); - assertTermDocsCount("after delete 1", reader2, searchTerm1, 0); - assertTermDocsCount("after delete 1", reader2, searchTerm2, 100); - assertTermDocsCount("after delete 1", reader2, searchTerm3, 100); - reader2.close(); - - // Make sure reader 1 is unchanged since it was open earlier - assertEquals("after delete 1", 100, reader1.docFreq(searchTerm1)); - assertEquals("after delete 1", 100, reader1.docFreq(searchTerm2)); - assertEquals("after delete 1", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("after delete 1", reader1, searchTerm1, 100); - assertTermDocsCount("after delete 1", reader1, searchTerm2, 100); - assertTermDocsCount("after delete 1", reader1, searchTerm3, 100); - - - // ATTEMPT TO DELETE FROM STALE READER - // delete documents containing term: bbb - try { - reader1.deleteDocuments(searchTerm2); - fail("Delete allowed from a stale index reader"); - } catch (IOException e) { - /* success */ - } - - // RECREATE READER AND TRY AGAIN - reader1.close(); - reader1 = IndexReader.open(dir, false); - assertEquals("reopened", 100, reader1.docFreq(searchTerm1)); - assertEquals("reopened", 100, reader1.docFreq(searchTerm2)); - assertEquals("reopened", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("reopened", reader1, searchTerm1, 0); - assertTermDocsCount("reopened", reader1, searchTerm2, 100); - assertTermDocsCount("reopened", reader1, searchTerm3, 100); - - reader1.deleteDocuments(searchTerm2); - assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1)); - assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2)); - assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("deleted 2", reader1, searchTerm1, 0); - assertTermDocsCount("deleted 2", reader1, searchTerm2, 0); - assertTermDocsCount("deleted 2", reader1, searchTerm3, 100); - reader1.close(); - - // Open another reader to confirm that everything is deleted - reader2 = IndexReader.open(dir, false); - assertTermDocsCount("reopened 2", reader2, searchTerm1, 0); - assertTermDocsCount("reopened 2", reader2, searchTerm2, 0); - assertTermDocsCount("reopened 2", reader2, searchTerm3, 100); - reader2.close(); - - dir.close(); - } - - private void addDocumentWithFields(IndexWriter writer) throws IOException + static void addDocumentWithFields(IndexWriter writer) throws IOException { Document doc = new Document(); doc.add(newField("keyword","test1", Field.Store.YES, Field.Index.NOT_ANALYZED)); @@ -1297,7 +785,7 @@ writer.addDocument(doc); } - private void addDocumentWithDifferentFields(IndexWriter writer) throws IOException + static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException { Document doc = new Document(); doc.add(newField("keyword2","test1", Field.Store.YES, Field.Index.NOT_ANALYZED)); @@ -1307,7 +795,7 @@ writer.addDocument(doc); } - private void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException + static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException { Document doc = new Document(); doc.add(newField("tvnot","tvnot", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); @@ -1319,7 +807,7 @@ writer.addDocument(doc); } - private void addDoc(IndexWriter writer, String value) throws IOException { + static void addDoc(IndexWriter writer, String value) throws IOException { Document doc = new Document(); doc.add(newField("content", value, Field.Store.NO, Field.Index.ANALYZED)); writer.addDocument(doc); @@ -1557,28 +1045,7 @@ dir.close(); } - // LUCENE-1647 - public void testIndexReaderUnDeleteAll() throws Exception { - MockDirectoryWrapper dir = newDirectory(); - dir.setPreventDoubleWrite(false); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - writer.addDocument(createDocument("a")); - writer.addDocument(createDocument("b")); - writer.addDocument(createDocument("c")); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocuments(new Term("id", "a")); - reader.flush(); - reader.deleteDocuments(new Term("id", "b")); - reader.undeleteAll(); - reader.deleteDocuments(new Term("id", "b")); - reader.close(); - IndexReader.open(dir,true).close(); - dir.close(); - } - - private Document createDocument(String id) { + static Document createDocument(String id) { Document doc = new Document(); doc.add(newField("id", id, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); return doc; @@ -1692,54 +1159,6 @@ dir.close(); } - // LUCENE-1579: Make sure all SegmentReaders are new when - // reopen switches readOnly - public void testReopenChangeReadonly() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter( - dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setMaxBufferedDocs(-1). - setMergePolicy(newLogMergePolicy(10)) - ); - Document doc = new Document(); - doc.add(newField("number", "17", Field.Store.NO, Field.Index.NOT_ANALYZED)); - writer.addDocument(doc); - writer.commit(); - - // Open reader1 - IndexReader r = IndexReader.open(dir, false); - assertTrue(r instanceof DirectoryReader); - IndexReader r1 = getOnlySegmentReader(r); - final int[] ints = FieldCache.DEFAULT.getInts(r1, "number"); - assertEquals(1, ints.length); - assertEquals(17, ints[0]); - - // Reopen to readonly w/ no chnages - IndexReader r3 = r.reopen(true); - assertTrue(((DirectoryReader) r3).readOnly); - r3.close(); - - // Add new segment - writer.addDocument(doc); - writer.commit(); - - // Reopen reader1 --> reader2 - IndexReader r2 = r.reopen(true); - r.close(); - assertTrue(((DirectoryReader) r2).readOnly); - IndexReader[] subs = r2.getSequentialSubReaders(); - final int[] ints2 = FieldCache.DEFAULT.getInts(subs[0], "number"); - r2.close(); - - assertTrue(((SegmentReader) subs[0]).readOnly); - assertTrue(((SegmentReader) subs[1]).readOnly); - assertTrue(ints == ints2); - - writer.close(); - dir.close(); - } - // LUCENE-1586: getUniqueTermCount public void testUniqueTermCount() throws Exception { Directory dir = newDirectory(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (working copy) @@ -114,7 +114,6 @@ // Make sure reader can open the index: IndexReader.open(dir, true).close(); - _TestUtil.checkIndex(dir); } dir.close(); @@ -491,7 +490,6 @@ w.addDocument(doc); w.close(); - _TestUtil.checkIndex(dir); dir.close(); } Index: lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (working copy) @@ -42,7 +42,7 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { public void testGlobalFieldNumberFiles() throws IOException { - for (int i = 0; i < 39; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13); i++) { Directory dir = newDirectory(); { IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, @@ -113,7 +113,7 @@ } public void testIndexReaderCommit() throws IOException { - for (int i = 0; i < 39; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13); i++) { Directory dir = newDirectory(); { IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, @@ -156,7 +156,7 @@ } public void testGlobalFieldNumberFilesAcrossCommits() throws IOException { - for (int i = 0; i < 39; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13); i++) { Directory dir = newDirectory(); { IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( @@ -207,7 +207,7 @@ } public void testGlobalFieldNumberOnOldCommit() throws IOException { - for (int i = 0; i < 39; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 39 : 13); i++) { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy( Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -133,14 +133,14 @@ dir.close(); } - private void addDoc(IndexWriter writer) throws IOException + static void addDoc(IndexWriter writer) throws IOException { Document doc = new Document(); doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED)); writer.addDocument(doc); } - private void addDocWithIndex(IndexWriter writer, int index) throws IOException + static void addDocWithIndex(IndexWriter writer, int index) throws IOException { Document doc = new Document(); doc.add(newField("content", "aaa " + index, Field.Store.YES, Field.Index.ANALYZED)); @@ -165,142 +165,6 @@ } } - public void testOptimizeMaxNumSegments() throws IOException { - - MockDirectoryWrapper dir = newDirectory(); - - final Document doc = new Document(); - doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED)); - final int incrMin = TEST_NIGHTLY ? 15 : 40; - for(int numDocs=10;numDocs<500;numDocs += _TestUtil.nextInt(random, incrMin, 5*incrMin)) { - LogDocMergePolicy ldmp = new LogDocMergePolicy(); - ldmp.setMinMergeDocs(1); - ldmp.setMergeFactor(5); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(2).setMergePolicy( - ldmp)); - for(int j=0;j= 20 (maxMergeDocs) docs - private class MyMergeScheduler extends MergeScheduler { - @Override - synchronized public void merge(IndexWriter writer) - throws CorruptIndexException, IOException { - - while(true) { - MergePolicy.OneMerge merge = writer.getNextMerge(); - if (merge == null) { - break; - } - for(int i=0;i> 10) + UnicodeUtil.UNI_SUR_HIGH_START); - chars[len++] = (char) (((ch-0x0010000) & 0x3FFL) + UnicodeUtil.UNI_SUR_LOW_START); - } - - UnicodeUtil.UTF16toUTF8(chars, 0, len, utf8); - - String s1 = new String(chars, 0, len); - String s2 = new String(utf8.bytes, 0, utf8.length, "UTF-8"); - assertEquals("codepoint " + ch, s1, s2); - - UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16); - assertEquals("codepoint " + ch, s1, new String(utf16.chars, 0, utf16.length)); - - byte[] b = s1.getBytes("UTF-8"); - assertEquals(utf8.length, b.length); - for(int j=0;j 0 && buffer[offset] >= 0xdc00 && buffer[offset] < 0xe000) - // Don't start in the middle of a valid surrogate pair - offset--; - - for(int i=offset;i data = new HashMap(); - data.put("label", "test1"); - w.commit(data); - w.close(); - - assertEquals("test1", IndexReader.getCommitUserData(dir).get("label")); - - r = IndexReader.open(dir, true); - assertEquals("test1", r.getCommitUserData().get("label")); - r.close(); - - w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); - w.optimize(); - w.close(); - - assertEquals("test1", IndexReader.getCommitUserData(dir).get("label")); - - dir.close(); - } - - // LUCENE-2529 public void testPositionIncrementGapEmptyField() throws Exception { Directory dir = newDirectory(); @@ -2299,24 +1297,6 @@ d.close(); } - public void testEmbeddedFFFF() throws Throwable { - - Directory d = newDirectory(); - IndexWriter w = new IndexWriter(d, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); - Document doc = new Document(); - doc.add(newField("field", "a a\uffffb", Field.Store.NO, Field.Index.ANALYZED)); - w.addDocument(doc); - doc = new Document(); - doc.add(newField("field", "a", Field.Store.NO, Field.Index.ANALYZED)); - w.addDocument(doc); - IndexReader r = w.getReader(); - assertEquals(1, r.docFreq(new Term("field", "a\uffffb"))); - r.close(); - w.close(); - _TestUtil.checkIndex(d); - d.close(); - } - public void testNoDocsIndex() throws Throwable { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( @@ -2326,188 +1306,9 @@ writer.addDocument(new Document()); writer.close(); - _TestUtil.checkIndex(dir); dir.close(); } - // LUCENE-2095: make sure with multiple threads commit - // doesn't return until all changes are in fact in the - // index - public void testCommitThreadSafety() throws Throwable { - final int NUM_THREADS = 5; - final double RUN_SEC = 0.5; - final Directory dir = newDirectory(); - final RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - _TestUtil.reduceOpenFiles(w.w); - w.commit(); - final AtomicBoolean failed = new AtomicBoolean(); - Thread[] threads = new Thread[NUM_THREADS]; - final long endTime = System.currentTimeMillis()+((long) (RUN_SEC*1000)); - for(int i=0;i allTerms, boolean isTop) throws IOException { - TermsEnum terms = MultiFields.getFields(r).terms("f").iterator(); - - BytesRef last = new BytesRef(); - - Set seenTerms = new HashSet(); - - while(true) { - final BytesRef term = terms.next(); - if (term == null) { - break; - } - - assertTrue(last.compareTo(term) < 0); - last.copy(term); - - final String s = term.utf8ToString(); - assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s)); - seenTerms.add(s); - } - - if (isTop) { - assertTrue(allTerms.equals(seenTerms)); - } - - // Test seeking: - Iterator it = seenTerms.iterator(); - while(it.hasNext()) { - BytesRef tr = new BytesRef(it.next()); - assertEquals("seek failed for term=" + termDesc(tr.utf8ToString()), - TermsEnum.SeekStatus.FOUND, - terms.seek(tr)); - } - } - - private final String asUnicodeChar(char c) { - return "U+" + Integer.toHexString(c); - } - - private final String termDesc(String s) { - final String s0; - assertTrue(s.length() <= 2); - if (s.length() == 1) { - s0 = asUnicodeChar(s.charAt(0)); - } else { - s0 = asUnicodeChar(s.charAt(0)) + "," + asUnicodeChar(s.charAt(1)); - } - return s0; - } - - // Make sure terms, including ones with surrogate pairs, - // sort in codepoint sort order by default - public void testTermUTF16SortOrder() throws Throwable { - Random rnd = random; - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(rnd, dir); - Document d = new Document(); - // Single segment - Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED); - d.add(f); - char[] chars = new char[2]; - final Set allTerms = new HashSet(); - - int num = 200 * RANDOM_MULTIPLIER; - for (int i = 0; i < num; i++) { - - final String s; - if (rnd.nextBoolean()) { - // Single char - if (rnd.nextBoolean()) { - // Above surrogates - chars[0] = (char) getInt(rnd, 1+UnicodeUtil.UNI_SUR_LOW_END, 0xffff); - } else { - // Below surrogates - chars[0] = (char) getInt(rnd, 0, UnicodeUtil.UNI_SUR_HIGH_START-1); - } - s = new String(chars, 0, 1); - } else { - // Surrogate pair - chars[0] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_HIGH_START, UnicodeUtil.UNI_SUR_HIGH_END); - assertTrue(((int) chars[0]) >= UnicodeUtil.UNI_SUR_HIGH_START && ((int) chars[0]) <= UnicodeUtil.UNI_SUR_HIGH_END); - chars[1] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_LOW_START, UnicodeUtil.UNI_SUR_LOW_END); - s = new String(chars, 0, 2); - } - allTerms.add(s); - f.setValue(s); - - writer.addDocument(d); - - if ((1+i) % 42 == 0) { - writer.commit(); - } - } - - IndexReader r = writer.getReader(); - - // Test each sub-segment - final IndexReader[] subs = r.getSequentialSubReaders(); - for(int i=0;i commitData = new HashMap(); - commitData.put("tag", "first"); - w.commit(commitData); - - // commit to "second" - w.addDocument(doc); - commitData.put("tag", "second"); - w.commit(commitData); - w.close(); - - // open "first" with IndexWriter - IndexCommit commit = null; - for(IndexCommit c : IndexReader.listCommits(dir)) { - if (c.getUserData().get("tag").equals("first")) { - commit = c; - break; - } - } - - assertNotNull(commit); - - w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit)); - - assertEquals(1, w.numDocs()); - - // commit IndexWriter to "third" - w.addDocument(doc); - commitData.put("tag", "third"); - w.commit(commitData); - w.close(); - - // make sure "second" commit is still there - commit = null; - for(IndexCommit c : IndexReader.listCommits(dir)) { - if (c.getUserData().get("tag").equals("second")) { - commit = c; - break; - } - } - - assertNotNull(commit); - - IndexReader r = IndexReader.open(commit, true); - assertEquals(2, r.numDocs()); - r.close(); - - // open "second", w/ writeable IndexReader & commit - r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false); - assertEquals(2, r.numDocs()); - r.deleteDocument(0); - r.deleteDocument(1); - commitData.put("tag", "fourth"); - r.commit(commitData); - r.close(); - - // make sure "third" commit is still there - commit = null; - for(IndexCommit c : IndexReader.listCommits(dir)) { - if (c.getUserData().get("tag").equals("third")) { - commit = c; - break; - } - } - assertNotNull(commit); - - dir.close(); - } - public void testRandomStoredFields() throws IOException { Directory dir = newDirectory(); Random rand = random; @@ -2980,39 +1644,7 @@ dir.close(); } - public void testDeleteAllSlowly() throws Exception { - final Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random, dir); - final int NUM_DOCS = 1000 * RANDOM_MULTIPLIER; - final List ids = new ArrayList(NUM_DOCS); - for(int id=0;id terms = new HashSet(); while(terms.size() < NUM_TERMS) { final String s = _TestUtil.randomRealisticUnicodeString(random); @@ -226,7 +226,7 @@ final BytesRef[] termsArray = terms.toArray(new BytesRef[terms.size()]); Arrays.sort(termsArray); - final int NUM_DOCS = 1000 * RANDOM_MULTIPLIER; + final int NUM_DOCS = (TEST_NIGHTLY ? 1000 : 100) * RANDOM_MULTIPLIER; IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); @@ -264,7 +264,7 @@ } for(int ord : ordsForDocSet) { ordsForDoc[upto++] = ord; - Field field = newField("field", termsArray[ord].utf8ToString(), Field.Index.NOT_ANALYZED); + Field field = newField("field", termsArray[ord].utf8ToString(), Field.Index.NOT_ANALYZED_NO_NORMS); if (VERBOSE) { System.out.println(" f=" + termsArray[ord].utf8ToString()); } @@ -317,7 +317,7 @@ } final String[] prefixesArray = prefixes.toArray(new String[prefixes.size()]); - final int NUM_TERMS = 100 * RANDOM_MULTIPLIER; + final int NUM_TERMS = (TEST_NIGHTLY ? 100 : 20) * RANDOM_MULTIPLIER; final Set terms = new HashSet(); while(terms.size() < NUM_TERMS) { final String s = prefixesArray[random.nextInt(prefixesArray.length)] + _TestUtil.randomRealisticUnicodeString(random); @@ -329,7 +329,7 @@ final BytesRef[] termsArray = terms.toArray(new BytesRef[terms.size()]); Arrays.sort(termsArray); - final int NUM_DOCS = 1000 * RANDOM_MULTIPLIER; + final int NUM_DOCS = (TEST_NIGHTLY ? 1000 : 100) * RANDOM_MULTIPLIER; IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); @@ -367,7 +367,7 @@ } for(int ord : ordsForDocSet) { ordsForDoc[upto++] = ord; - Field field = newField("field", termsArray[ord].utf8ToString(), Field.Index.NOT_ANALYZED); + Field field = newField("field", termsArray[ord].utf8ToString(), Field.Index.NOT_ANALYZED_NO_NORMS); if (VERBOSE) { System.out.println(" f=" + termsArray[ord].utf8ToString()); } Index: lucene/src/test/org/apache/lucene/index/TestOmitNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestOmitNorms.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestOmitNorms.java (working copy) @@ -64,7 +64,6 @@ writer.optimize(); // flush writer.close(); - _TestUtil.checkIndex(ram); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); FieldInfos fi = reader.fieldInfos(); @@ -121,8 +120,6 @@ // flush writer.close(); - _TestUtil.checkIndex(ram); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); FieldInfos fi = reader.fieldInfos(); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms); @@ -170,8 +167,6 @@ // flush writer.close(); - _TestUtil.checkIndex(ram); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); FieldInfos fi = reader.fieldInfos(); assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitNorms); @@ -218,7 +213,6 @@ writer.close(); assertNoNrm(ram); - _TestUtil.checkIndex(ram); ram.close(); } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (working copy) @@ -821,7 +821,6 @@ writer.close(); - _TestUtil.checkIndex(dir1); dir1.close(); } @@ -908,7 +907,6 @@ assertEquals(0, excs.size()); writer.close(); - _TestUtil.checkIndex(dir1); r.close(); dir1.close(); } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy) @@ -18,11 +18,21 @@ */ import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -860,4 +870,79 @@ modifier.close(); dir.close(); } + + public void testDeleteAllSlowly() throws Exception { + final Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random, dir); + final int NUM_DOCS = 1000 * RANDOM_MULTIPLIER; + final List ids = new ArrayList(NUM_DOCS); + for(int id=0;id lazyFieldNames = new HashSet(); lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY); SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(Collections. emptySet(), lazyFieldNames); Index: lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (working copy) @@ -36,7 +36,7 @@ final LineFileDocs docs = new LineFileDocs(random); final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - final int SIZE = 200 * RANDOM_MULTIPLIER; + final int SIZE = (TEST_NIGHTLY ? 200 : 20) * RANDOM_MULTIPLIER; int id = 0; IndexReader r = null; final int numUpdates = (int) (SIZE * (2+random.nextDouble())); @@ -82,9 +82,8 @@ for (int r = 0; r < 3; r++) { final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2)); - final int SIZE = 200 * RANDOM_MULTIPLIER; - final int numUpdates = (int) (SIZE * (2 + random.nextDouble())); - int numThreads = 3 + random.nextInt(12); + final int numUpdates = (TEST_NIGHTLY ? 200 : 20) * RANDOM_MULTIPLIER; + int numThreads = _TestUtil.nextInt(random, 2, 6); IndexingThread[] threads = new IndexingThread[numThreads]; for (int i = 0; i < numThreads; i++) { threads[i] = new IndexingThread(docs, w, numUpdates); @@ -97,6 +96,7 @@ w.close(); } + IndexReader open = IndexReader.open(dir); assertEquals(1, open.numDocs()); open.close(); @@ -123,9 +123,10 @@ Document doc = new Document();// docs.nextDoc(); doc.add(newField("id", "test", Index.NOT_ANALYZED)); writer.updateDocument(new Term("id", "test"), doc); - if (random.nextInt(10) == 0) { - if (open == null) + if (random.nextInt(3) == 0) { + if (open == null) { open = IndexReader.open(writer, true); + } IndexReader reader = open.reopen(); if (reader != open) { open.close(); @@ -134,11 +135,12 @@ assertEquals("iter: " + i + " numDocs: "+ open.numDocs() + " del: " + open.numDeletedDocs() + " max: " + open.maxDoc(), 1, open.numDocs()); } } - open.close(); + if (open != null) { + open.close(); + } } catch (Exception e) { - fail(e.getMessage()); + throw new RuntimeException(e); } - } } } Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -36,6 +36,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Similarity; @@ -1244,4 +1245,52 @@ r.close(); dir.close(); } + + // LUCENE-1579: Make sure all SegmentReaders are new when + // reopen switches readOnly + public void testReopenChangeReadonly() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDocs(-1). + setMergePolicy(newLogMergePolicy(10)) + ); + Document doc = new Document(); + doc.add(newField("number", "17", Field.Store.NO, Field.Index.NOT_ANALYZED)); + writer.addDocument(doc); + writer.commit(); + + // Open reader1 + IndexReader r = IndexReader.open(dir, false); + assertTrue(r instanceof DirectoryReader); + IndexReader r1 = getOnlySegmentReader(r); + final int[] ints = FieldCache.DEFAULT.getInts(r1, "number"); + assertEquals(1, ints.length); + assertEquals(17, ints[0]); + + // Reopen to readonly w/ no chnages + IndexReader r3 = r.reopen(true); + assertTrue(((DirectoryReader) r3).readOnly); + r3.close(); + + // Add new segment + writer.addDocument(doc); + writer.commit(); + + // Reopen reader1 --> reader2 + IndexReader r2 = r.reopen(true); + r.close(); + assertTrue(((DirectoryReader) r2).readOnly); + IndexReader[] subs = r2.getSequentialSubReaders(); + final int[] ints2 = FieldCache.DEFAULT.getInts(subs[0], "number"); + r2.close(); + + assertTrue(((SegmentReader) subs[0]).readOnly); + assertTrue(((SegmentReader) subs[1]).readOnly); + assertTrue(ints == ints2); + + writer.close(); + dir.close(); + } } Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -816,8 +816,6 @@ c.joinThreads(); - _TestUtil.checkIndex(c.dir2); - c.closeDir(); assertTrue(c.failures.size() == 0); @@ -908,8 +906,6 @@ if (VERBOSE) { System.out.println("TEST: done join threads"); } - _TestUtil.checkIndex(c.dir2); - c.closeDir(); assertTrue(c.failures.size() == 0); @@ -933,8 +929,6 @@ c.joinThreads(); - _TestUtil.checkIndex(c.dir2); - c.closeDir(); assertTrue(c.failures.size() == 0); @@ -1039,7 +1033,6 @@ writer.addIndexes(aux, aux2); assertEquals(190, writer.maxDoc()); writer.close(); - _TestUtil.checkIndex(dir, provider); dir.close(); aux.close(); Index: lucene/src/test/org/apache/lucene/index/TestPayloads.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestPayloads.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestPayloads.java (working copy) @@ -615,8 +615,6 @@ writer.optimize(); writer.close(); - _TestUtil.checkIndex(dir); - dir.close(); } } Index: lucene/src/test/org/apache/lucene/index/TestLazyBug.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLazyBug.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestLazyBug.java (working copy) @@ -25,6 +25,8 @@ import org.apache.lucene.document.*; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; /** @@ -33,8 +35,8 @@ */ public class TestLazyBug extends LuceneTestCase { - public static int NUM_DOCS = 500; - public static int NUM_FIELDS = 100; + public static int NUM_DOCS = TEST_NIGHTLY ? 500 : 50; + public static int NUM_FIELDS = TEST_NIGHTLY ? 100 : 10; private static String[] data = new String[] { "now", @@ -49,6 +51,19 @@ private static Set dataset = asSet(data); private static String MAGIC_FIELD = "f"+(NUM_FIELDS/3); + + private static Directory directory; + + @BeforeClass + public static void beforeClass() throws Exception { + directory = makeIndex(); + } + + @AfterClass + public static void afterClass() throws Exception { + directory.close(); + directory = null; + } private static FieldSelector SELECTOR = new FieldSelector() { public FieldSelectorResult accept(String f) { @@ -59,7 +74,7 @@ } }; - private Directory makeIndex() throws Exception { + private static Directory makeIndex() throws Exception { Directory dir = newDirectory(); try { IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( @@ -72,7 +87,7 @@ doc.add(newField("f"+f, data[f % data.length] + '#' + data[random.nextInt(data.length)], - Field.Store.YES, + Field.Store.NO, Field.Index.ANALYZED)); } writer.addDocument(doc); @@ -85,8 +100,7 @@ } public void doTest(int[] docs) throws Exception { - Directory dir = makeIndex(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(directory, true); for (int i = 0; i < docs.length; i++) { Document d = reader.document(docs[i], SELECTOR); d.get(MAGIC_FIELD); @@ -109,7 +123,6 @@ } } reader.close(); - dir.close(); } public void testLazyWorks() throws Exception { Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (working copy) @@ -19,6 +19,9 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.util.LuceneTestCase; @@ -113,4 +116,187 @@ } writer.close(); } + + // LUCENE-325: test expungeDeletes, when 2 singular merges + // are required + public void testExpungeDeletes() throws IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMaxBufferedDocs(2).setRAMBufferSizeMB( + IndexWriterConfig.DISABLE_AUTO_FLUSH)); + writer.setInfoStream(VERBOSE ? System.out : null); + Document document = new Document(); + + document = new Document(); + Field storedField = newField("stored", "stored", Field.Store.YES, + Field.Index.NO); + document.add(storedField); + Field termVectorField = newField("termVector", "termVector", + Field.Store.NO, Field.Index.NOT_ANALYZED, + Field.TermVector.WITH_POSITIONS_OFFSETS); + document.add(termVectorField); + for(int i=0;i<10;i++) + writer.addDocument(document); + writer.close(); + + IndexReader ir = IndexReader.open(dir, false); + assertEquals(10, ir.maxDoc()); + assertEquals(10, ir.numDocs()); + ir.deleteDocument(0); + ir.deleteDocument(7); + assertEquals(8, ir.numDocs()); + ir.close(); + + writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); + assertEquals(8, writer.numDocs()); + assertEquals(10, writer.maxDoc()); + writer.expungeDeletes(); + assertEquals(8, writer.numDocs()); + writer.close(); + ir = IndexReader.open(dir, true); + assertEquals(8, ir.maxDoc()); + assertEquals(8, ir.numDocs()); + ir.close(); + dir.close(); + } + + // LUCENE-325: test expungeDeletes, when many adjacent merges are required + public void testExpungeDeletes2() throws IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDocs(2). + setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH). + setMergePolicy(newLogMergePolicy(50)) + ); + + Document document = new Document(); + + document = new Document(); + Field storedField = newField("stored", "stored", Store.YES, + Index.NO); + document.add(storedField); + Field termVectorField = newField("termVector", "termVector", + Store.NO, Index.NOT_ANALYZED, + TermVector.WITH_POSITIONS_OFFSETS); + document.add(termVectorField); + for(int i=0;i<98;i++) + writer.addDocument(document); + writer.close(); + + IndexReader ir = IndexReader.open(dir, false); + assertEquals(98, ir.maxDoc()); + assertEquals(98, ir.numDocs()); + for(int i=0;i<98;i+=2) + ir.deleteDocument(i); + assertEquals(49, ir.numDocs()); + ir.close(); + + writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMergePolicy(newLogMergePolicy(3)) + ); + assertEquals(49, writer.numDocs()); + writer.expungeDeletes(); + writer.close(); + ir = IndexReader.open(dir, true); + assertEquals(49, ir.maxDoc()); + assertEquals(49, ir.numDocs()); + ir.close(); + dir.close(); + } + + // LUCENE-325: test expungeDeletes without waiting, when + // many adjacent merges are required + public void testExpungeDeletes3() throws IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDocs(2). + setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH). + setMergePolicy(newLogMergePolicy(50)) + ); + + Document document = new Document(); + + document = new Document(); + Field storedField = newField("stored", "stored", Field.Store.YES, + Field.Index.NO); + document.add(storedField); + Field termVectorField = newField("termVector", "termVector", + Field.Store.NO, Field.Index.NOT_ANALYZED, + Field.TermVector.WITH_POSITIONS_OFFSETS); + document.add(termVectorField); + for(int i=0;i<98;i++) + writer.addDocument(document); + writer.close(); + + IndexReader ir = IndexReader.open(dir, false); + assertEquals(98, ir.maxDoc()); + assertEquals(98, ir.numDocs()); + for(int i=0;i<98;i+=2) + ir.deleteDocument(i); + assertEquals(49, ir.numDocs()); + ir.close(); + + writer = new IndexWriter( + dir, + newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMergePolicy(newLogMergePolicy(3)) + ); + writer.expungeDeletes(false); + writer.close(); + ir = IndexReader.open(dir, true); + assertEquals(49, ir.maxDoc()); + assertEquals(49, ir.numDocs()); + ir.close(); + dir.close(); + } + + // Just intercepts all merges & verifies that we are never + // merging a segment with >= 20 (maxMergeDocs) docs + private class MyMergeScheduler extends MergeScheduler { + @Override + synchronized public void merge(IndexWriter writer) + throws CorruptIndexException, IOException { + + while(true) { + MergePolicy.OneMerge merge = writer.getNextMerge(); + if (merge == null) { + break; + } + for(int i=0;i uniqueTerms = new HashSet(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (working copy) @@ -28,6 +28,7 @@ import java.util.List; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.Constants; import org.apache.lucene.util._TestUtil; @@ -123,7 +124,8 @@ */ public boolean checkIndexes(File file) throws IOException { if (file.isDirectory()) { - Directory dir = newFSDirectory(file); + MockDirectoryWrapper dir = newFSDirectory(file); + dir.setCheckIndexOnClose(false); // don't double-checkindex if (IndexReader.indexExists(dir)) { if (VERBOSE) { System.err.println("Checking index: " + file); Index: lucene/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java (working copy) @@ -62,7 +62,6 @@ iwOut.optimize(); iwOut.close(); - _TestUtil.checkIndex(rdOut); rdOut.close(); rd1.close(); rd2.close(); @@ -122,7 +121,6 @@ iwOut.optimize(); iwOut.close(); - _TestUtil.checkIndex(rdOut); rdOut.close(); } } Index: lucene/src/test/org/apache/lucene/index/TestLongPostings.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLongPostings.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestLongPostings.java (working copy) @@ -70,7 +70,7 @@ // randomness (ie same seed will point to same dir): Directory dir = newFSDirectory(_TestUtil.getTempDir("longpostings" + "." + random.nextLong())); - final int NUM_DOCS = (int) ((TEST_NIGHTLY ? 4e6 : (RANDOM_MULTIPLIER*2e4)) * (1+random.nextDouble())); + final int NUM_DOCS = (int) ((TEST_NIGHTLY ? 4e6 : (RANDOM_MULTIPLIER*2e3)) * (1+random.nextDouble())); if (VERBOSE) { System.out.println("TEST: NUM_DOCS=" + NUM_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java (working copy) @@ -231,8 +231,8 @@ for (int i = 0; i < numThreads.length; i++) { AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); MockDirectoryWrapper dir = newDirectory(); - // mock a very slow harddisk here so that flushing is very slow - dir.setThrottling(MockDirectoryWrapper.Throttling.ALWAYS); + // mock a very slow harddisk sometimes here so that flushing is very slow + dir.setThrottling(MockDirectoryWrapper.Throttling.SOMETIMES); IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -188,8 +188,6 @@ w.setInfoStream(VERBOSE ? System.out : null); w.optimize(); w.close(); - - _TestUtil.checkIndex(dir); dir.close(); _TestUtil.rmDir(oldIndxeDir); @@ -207,8 +205,6 @@ TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.addIndexes(dir); w.close(); - - _TestUtil.checkIndex(targetDir); dir.close(); targetDir.close(); @@ -229,9 +225,7 @@ w.addIndexes(reader); w.close(); reader.close(); - - _TestUtil.checkIndex(targetDir); - + dir.close(); targetDir.close(); _TestUtil.rmDir(oldIndxeDir); @@ -743,8 +737,6 @@ .upgrade(); checkAllSegmentsUpgraded(dir); - - _TestUtil.checkIndex(dir); dir.close(); _TestUtil.rmDir(oldIndxeDir); Index: lucene/src/test/org/apache/lucene/util/TestArrayUtil.java =================================================================== --- lucene/src/test/org/apache/lucene/util/TestArrayUtil.java (revision 1132463) +++ lucene/src/test/org/apache/lucene/util/TestArrayUtil.java (working copy) @@ -126,7 +126,8 @@ } public void testQuickSort() { - for (int i = 0, c = 500 * RANDOM_MULTIPLIER; i < c; i++) { + int num = (TEST_NIGHTLY ? 500 : 50) * RANDOM_MULTIPLIER; + for (int i = 0; i < num; i++) { Integer[] a1 = createRandomArray(1000), a2 = a1.clone(); ArrayUtil.quickSort(a1); Arrays.sort(a2); @@ -154,7 +155,8 @@ // This is a test for LUCENE-3054 (which fails without the merge sort fall back with stack overflow in most cases) public void testQuickToMergeSortFallback() { - for (int i = 0, c = 500 * RANDOM_MULTIPLIER; i < c; i++) { + int num = (TEST_NIGHTLY ? 500 : 50) * RANDOM_MULTIPLIER; + for (int i = 0; i < num; i++) { Integer[] a1 = createSparseRandomArray(40000), a2 = a1.clone(); ArrayUtil.quickSort(a1); Arrays.sort(a2); @@ -163,7 +165,8 @@ } public void testMergeSort() { - for (int i = 0, c = 500 * RANDOM_MULTIPLIER; i < c; i++) { + int num = (TEST_NIGHTLY ? 500 : 50) * RANDOM_MULTIPLIER; + for (int i = 0; i < num; i++) { Integer[] a1 = createRandomArray(1000), a2 = a1.clone(); ArrayUtil.mergeSort(a1); Arrays.sort(a2); Index: lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (revision 1132463) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (working copy) @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Random; +import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.automaton.CharacterRunAutomaton; /** @@ -127,13 +128,16 @@ private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) { Integer val = previousMappings.get(fieldName); if (val == null) { - switch(random.nextInt(3)) { - case 0: val = -1; // no payloads - break; - case 1: val = Integer.MAX_VALUE; // variable length payload - break; - case 2: val = random.nextInt(12); // fixed length payload - break; + val = -1; // no payloads + if (LuceneTestCase.TEST_NIGHTLY || random.nextInt(20) == 0) { + switch(random.nextInt(3)) { + case 0: val = -1; // no payloads + break; + case 1: val = Integer.MAX_VALUE; // variable length payload + break; + case 2: val = random.nextInt(12); // fixed length payload + break; + } } previousMappings.put(fieldName, val); // save it so we are consistent for this field } Index: lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (revision 1132463) +++ lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (working copy) @@ -272,7 +272,7 @@ public void close() throws IOException { // if someone isn't using getReader() API, we want to be sure to // maybeOptimize since presumably they might open a reader on the dir. - if (getReaderCalled == false && r.nextInt(4) == 2) { + if (getReaderCalled == false && r.nextInt(8) == 2) { doRandomOptimize(); } w.close(); Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision 1132463) +++ lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -836,14 +836,22 @@ c.setMergeScheduler(new SerialMergeScheduler()); } if (r.nextBoolean()) { - if (r.nextInt(20) == 17) { - c.setMaxBufferedDocs(2); + if ((TEST_NIGHTLY && random.nextBoolean()) || r.nextInt(20) == 17) { + // crazy value + c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 7)); } else { - c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 1000)); + // reasonable value + c.setMaxBufferedDocs(_TestUtil.nextInt(r, 8, 1000)); } } if (r.nextBoolean()) { - c.setTermIndexInterval(_TestUtil.nextInt(r, 1, 1000)); + if ((TEST_NIGHTLY && random.nextBoolean()) || r.nextInt(20) == 17) { + // crazy value + c.setTermIndexInterval(random.nextBoolean() ? _TestUtil.nextInt(r, 1, 31) : _TestUtil.nextInt(r, 129, 1000)); + } else { + // reasonable value + c.setTermIndexInterval(_TestUtil.nextInt(r, 32, 128)); + } } if (r.nextBoolean()) { c.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(_TestUtil.nextInt(r, 1, 20))); @@ -874,22 +882,22 @@ LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); logmp.setUseCompoundFile(r.nextBoolean()); logmp.setCalibrateSizeByDeletes(r.nextBoolean()); - if (r.nextInt(3) == 2) { - logmp.setMergeFactor(2); + if ((TEST_NIGHTLY && random.nextBoolean()) || r.nextInt(20) == 17) { + logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 4)); } else { - logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20)); + logmp.setMergeFactor(_TestUtil.nextInt(r, 5, 50)); } return logmp; } public static TieredMergePolicy newTieredMergePolicy(Random r) { TieredMergePolicy tmp = new TieredMergePolicy(); - if (r.nextInt(3) == 2) { - tmp.setMaxMergeAtOnce(2); - tmp.setMaxMergeAtOnceExplicit(2); + if ((TEST_NIGHTLY && random.nextBoolean()) || r.nextInt(20) == 17) { + tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 4)); + tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 4)); } else { - tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 20)); - tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 30)); + tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 5, 50)); + tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 5, 50)); } tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); @@ -1052,8 +1060,13 @@ /** Returns a new field instance, using the specified random. * See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */ public static Field newField(Random random, String name, String value, Store store, Index index, TermVector tv) { + if (!TEST_NIGHTLY && random.nextInt(20) > 0) { + // most of the time, don't modify the params + return new Field(name, value, store, index, tv); + } + if (!index.isIndexed()) - return new Field(name, value, store, index); + return new Field(name, value, store, index, tv); if (!store.isStored() && random.nextBoolean()) store = Store.YES; // randomly store it @@ -1115,7 +1128,7 @@ }; public static String randomDirectory(Random random) { - if (random.nextInt(10) == 0) { + if (random.nextInt(20) == 0) { return CORE_DIRECTORIES[random.nextInt(CORE_DIRECTORIES.length)]; } else { return "RAMDirectory"; @@ -1179,7 +1192,7 @@ public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException { if (random.nextBoolean()) { - if (maybeWrap && random.nextBoolean()) { + if (maybeWrap && random.nextInt(20) == 0) { return new IndexSearcher(new SlowMultiReaderWrapper(r)); } else { return new IndexSearcher(r); @@ -1408,6 +1421,10 @@ Codec codec = previousMappings.get(name); if (codec == null) { codec = knownCodecs.get(Math.abs(perFieldSeed ^ name.hashCode()) % knownCodecs.size()); + if (codec instanceof SimpleTextCodec && perFieldSeed % 5 != 0) { + // make simpletext rarer, choose again + codec = knownCodecs.get(Math.abs(perFieldSeed ^ name.toUpperCase(Locale.ENGLISH).hashCode()) % knownCodecs.size()); + } previousMappings.put(name, codec); } return codec.name;