Index: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (revision 1243051) +++ lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (working copy) @@ -17,67 +17,51 @@ * limitations under the License. */ +import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; import java.util.Random; +import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.NoSuchDirectoryException; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.junit.Assume; public class TestDirectoryReader extends LuceneTestCase { - protected Directory dir; - private Document doc1; - private Document doc2; - protected SegmentReader [] readers = new SegmentReader[2]; - protected SegmentInfos sis; - - @Override - public void setUp() throws Exception { - super.setUp(); - dir = createDirectory(); - doc1 = new Document(); - doc2 = new Document(); + + public void testDocument() throws IOException { + SegmentReader [] readers = new SegmentReader[2]; + Directory dir = newDirectory(); + Document doc1 = new Document(); + Document doc2 = new Document(); DocHelper.setupDoc(doc1); DocHelper.setupDoc(doc2); DocHelper.writeDoc(random, dir, doc1); DocHelper.writeDoc(random, dir, doc2); - sis = new SegmentInfos(); - sis.read(dir); - } - - @Override - public void tearDown() throws Exception { - if (readers[0] != null) readers[0].close(); - if (readers[1] != null) readers[1].close(); - dir.close(); - super.tearDown(); - } - - protected Directory createDirectory() throws IOException { - return newDirectory(); - } - - protected IndexReader openReader() throws IOException { - IndexReader reader; - reader = IndexReader.open(dir); - assertTrue(reader instanceof DirectoryReader); - - assertTrue(dir != null); - assertTrue(sis != null); + DirectoryReader reader = DirectoryReader.open(dir); assertTrue(reader != null); + assertTrue(reader instanceof StandardDirectoryReader); - return reader; - } - - public void testDocument() throws IOException { - sis.read(dir); - IndexReader reader = openReader(); - assertTrue(reader != null); Document newDoc1 = reader.document(0); assertTrue(newDoc1 != null); assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size()); @@ -86,21 +70,13 @@ assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size()); Terms vector = reader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY); assertNotNull(vector); - // TODO: pretty sure this check makes zero sense TestSegmentReader.checkNorms(reader); + reader.close(); + if (readers[0] != null) readers[0].close(); + if (readers[1] != null) readers[1].close(); + dir.close(); } - public void testIsCurrent() throws IOException { - Directory ramDir=newDirectory(); - addDoc(random, ramDir, "test foo", true); - DirectoryReader reader = DirectoryReader.open(ramDir); - assertTrue(reader.isCurrent()); // just opened, must be current - addDoc(random, ramDir, "more text", false); - assertFalse(reader.isCurrent()); // has been modified, not current anymore - reader.close(); - ramDir.close(); - } - public void testMultiTermDocs() throws IOException { Directory ramDir1=newDirectory(); addDoc(random, ramDir1, "test foo", true); @@ -109,8 +85,8 @@ Directory ramDir3=newDirectory(); addDoc(random, ramDir3, "test wow", true); - IndexReader[] readers1 = new IndexReader[]{IndexReader.open(ramDir1), IndexReader.open(ramDir3)}; - IndexReader[] readers2 = new IndexReader[]{IndexReader.open(ramDir1), IndexReader.open(ramDir2), IndexReader.open(ramDir3)}; + IndexReader[] readers1 = new IndexReader[]{DirectoryReader.open(ramDir1), DirectoryReader.open(ramDir3)}; + IndexReader[] readers2 = new IndexReader[]{DirectoryReader.open(ramDir1), DirectoryReader.open(ramDir2), DirectoryReader.open(ramDir3)}; MultiReader mr2 = new MultiReader(readers1); MultiReader mr3 = new MultiReader(readers2); @@ -159,4 +135,956 @@ iw.addDocument(doc); iw.close(); } + + public void testIsCurrent() throws Exception { + Directory d = newDirectory(); + IndexWriter writer = new IndexWriter(d, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random))); + addDocumentWithFields(writer); + writer.close(); + // set up reader: + DirectoryReader reader = DirectoryReader.open(d); + assertTrue(reader.isCurrent()); + // modify index by adding another document: + writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); + addDocumentWithFields(writer); + writer.close(); + assertFalse(reader.isCurrent()); + // re-create index: + writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); + addDocumentWithFields(writer); + writer.close(); + assertFalse(reader.isCurrent()); + reader.close(); + d.close(); + } + + /** + * Tests the IndexReader.getFieldNames implementation + * @throws Exception on error + */ + public void testGetFieldNames() throws Exception { + Directory d = newDirectory(); + // set up writer + IndexWriter writer = new IndexWriter( + d, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + ); + + Document doc = new Document(); + + FieldType customType3 = new FieldType(); + customType3.setStored(true); + + doc.add(new Field("keyword", "test1", StringField.TYPE_STORED)); + doc.add(new Field("text", "test1", TextField.TYPE_STORED)); + doc.add(new Field("unindexed", "test1", customType3)); + doc.add(new TextField("unstored","test1")); + writer.addDocument(doc); + + writer.close(); + // set up reader + DirectoryReader reader = DirectoryReader.open(d); + FieldInfos fieldInfos = MultiFields.getMergedFieldInfos(reader); + assertNotNull(fieldInfos.fieldInfo("keyword")); + assertNotNull(fieldInfos.fieldInfo("text")); + assertNotNull(fieldInfos.fieldInfo("unindexed")); + assertNotNull(fieldInfos.fieldInfo("unstored")); + reader.close(); + // add more documents + writer = new IndexWriter( + d, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setOpenMode(OpenMode.APPEND). + setMergePolicy(newLogMergePolicy()) + ); + // want to get some more segments here + int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor(); + for (int i = 0; i < 5*mergeFactor; i++) { + doc = new Document(); + doc.add(new Field("keyword", "test1", StringField.TYPE_STORED)); + doc.add(new Field("text", "test1", TextField.TYPE_STORED)); + doc.add(new Field("unindexed", "test1", customType3)); + doc.add(new TextField("unstored","test1")); + writer.addDocument(doc); + } + // new fields are in some different segments (we hope) + for (int i = 0; i < 5*mergeFactor; i++) { + doc = new Document(); + doc.add(new Field("keyword2", "test1", StringField.TYPE_STORED)); + doc.add(new Field("text2", "test1", TextField.TYPE_STORED)); + doc.add(new Field("unindexed2", "test1", customType3)); + doc.add(new TextField("unstored2","test1")); + writer.addDocument(doc); + } + // new termvector fields + + FieldType customType5 = new FieldType(TextField.TYPE_STORED); + customType5.setStoreTermVectors(true); + FieldType customType6 = new FieldType(TextField.TYPE_STORED); + customType6.setStoreTermVectors(true); + customType6.setStoreTermVectorOffsets(true); + FieldType customType7 = new FieldType(TextField.TYPE_STORED); + customType7.setStoreTermVectors(true); + customType7.setStoreTermVectorPositions(true); + FieldType customType8 = new FieldType(TextField.TYPE_STORED); + customType8.setStoreTermVectors(true); + customType8.setStoreTermVectorOffsets(true); + customType8.setStoreTermVectorPositions(true); + + for (int i = 0; i < 5*mergeFactor; i++) { + doc = new Document(); + doc.add(new Field("tvnot", "tvnot", TextField.TYPE_STORED)); + doc.add(new Field("termvector", "termvector", customType5)); + doc.add(new Field("tvoffset", "tvoffset", customType6)); + doc.add(new Field("tvposition", "tvposition", customType7)); + doc.add(new Field("tvpositionoffset", "tvpositionoffset", customType8)); + writer.addDocument(doc); + } + + writer.close(); + + // verify fields again + reader = DirectoryReader.open(d); + fieldInfos = MultiFields.getMergedFieldInfos(reader); + + Collection allFieldNames = new HashSet(); + Collection indexedFieldNames = new HashSet(); + Collection notIndexedFieldNames = new HashSet(); + Collection tvFieldNames = new HashSet(); + + for(FieldInfo fieldInfo : fieldInfos) { + final String name = fieldInfo.name; + allFieldNames.add(name); + if (fieldInfo.isIndexed) { + indexedFieldNames.add(name); + } else { + notIndexedFieldNames.add(name); + } + if (fieldInfo.storeTermVector) { + tvFieldNames.add(name); + } + } + + assertTrue(allFieldNames.contains("keyword")); + assertTrue(allFieldNames.contains("text")); + assertTrue(allFieldNames.contains("unindexed")); + assertTrue(allFieldNames.contains("unstored")); + assertTrue(allFieldNames.contains("keyword2")); + assertTrue(allFieldNames.contains("text2")); + assertTrue(allFieldNames.contains("unindexed2")); + assertTrue(allFieldNames.contains("unstored2")); + assertTrue(allFieldNames.contains("tvnot")); + assertTrue(allFieldNames.contains("termvector")); + assertTrue(allFieldNames.contains("tvposition")); + assertTrue(allFieldNames.contains("tvoffset")); + assertTrue(allFieldNames.contains("tvpositionoffset")); + + // verify that only indexed fields were returned + assertEquals(11, indexedFieldNames.size()); // 6 original + the 5 termvector fields + assertTrue(indexedFieldNames.contains("keyword")); + assertTrue(indexedFieldNames.contains("text")); + assertTrue(indexedFieldNames.contains("unstored")); + assertTrue(indexedFieldNames.contains("keyword2")); + assertTrue(indexedFieldNames.contains("text2")); + assertTrue(indexedFieldNames.contains("unstored2")); + assertTrue(indexedFieldNames.contains("tvnot")); + assertTrue(indexedFieldNames.contains("termvector")); + assertTrue(indexedFieldNames.contains("tvposition")); + assertTrue(indexedFieldNames.contains("tvoffset")); + assertTrue(indexedFieldNames.contains("tvpositionoffset")); + + // verify that only unindexed fields were returned + assertEquals(2, notIndexedFieldNames.size()); // the following fields + assertTrue(notIndexedFieldNames.contains("unindexed")); + assertTrue(notIndexedFieldNames.contains("unindexed2")); + + // verify index term vector fields + assertEquals(tvFieldNames.toString(), 4, tvFieldNames.size()); // 4 field has term vector only + assertTrue(tvFieldNames.contains("termvector")); + + reader.close(); + d.close(); + } + +public void testTermVectors() throws Exception { + Directory d = newDirectory(); + // set up writer + IndexWriter writer = new IndexWriter( + d, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMergePolicy(newLogMergePolicy()) + ); + // want to get some more segments here + // new termvector fields + int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor(); + FieldType customType5 = new FieldType(TextField.TYPE_STORED); + customType5.setStoreTermVectors(true); + FieldType customType6 = new FieldType(TextField.TYPE_STORED); + customType6.setStoreTermVectors(true); + customType6.setStoreTermVectorOffsets(true); + FieldType customType7 = new FieldType(TextField.TYPE_STORED); + customType7.setStoreTermVectors(true); + customType7.setStoreTermVectorPositions(true); + FieldType customType8 = new FieldType(TextField.TYPE_STORED); + customType8.setStoreTermVectors(true); + customType8.setStoreTermVectorOffsets(true); + customType8.setStoreTermVectorPositions(true); + for (int i = 0; i < 5 * mergeFactor; i++) { + Document doc = new Document(); + doc.add(new Field("tvnot", "one two two three three three", TextField.TYPE_STORED)); + doc.add(new Field("termvector", "one two two three three three", customType5)); + doc.add(new Field("tvoffset", "one two two three three three", customType6)); + doc.add(new Field("tvposition", "one two two three three three", customType7)); + doc.add(new Field("tvpositionoffset", "one two two three three three", customType8)); + + writer.addDocument(doc); + } + writer.close(); + d.close(); } + +static void assertTermDocsCount(String msg, + IndexReader reader, + Term term, + int expected) + throws IOException { + DocsEnum tdocs = _TestUtil.docs(random, reader, + term.field(), + new BytesRef(term.text()), + MultiFields.getLiveDocs(reader), + null, + false); + int count = 0; + if (tdocs != null) { + while(tdocs.nextDoc()!= DocIdSetIterator.NO_MORE_DOCS) { + count++; + } + } + assertEquals(msg + ", count mismatch", expected, count); +} + + + public void testBinaryFields() throws IOException { + Directory dir = newDirectory(); + byte[] bin = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); + + for (int i = 0; i < 10; i++) { + addDoc(writer, "document number " + (i + 1)); + addDocumentWithFields(writer); + addDocumentWithDifferentFields(writer); + addDocumentWithTermVectorFields(writer); + } + writer.close(); + writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); + Document doc = new Document(); + doc.add(new StoredField("bin1", bin)); + doc.add(new TextField("junk", "junk text")); + writer.addDocument(doc); + writer.close(); + DirectoryReader reader = DirectoryReader.open(dir); + Document doc2 = reader.document(reader.maxDoc() - 1); + IndexableField[] fields = doc2.getFields("bin1"); + assertNotNull(fields); + assertEquals(1, fields.length); + IndexableField b1 = fields[0]; + assertTrue(b1.binaryValue() != null); + BytesRef bytesRef = b1.binaryValue(); + assertEquals(bin.length, bytesRef.length); + for (int i = 0; i < bin.length; i++) { + assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]); + } + reader.close(); + // force merge + + + writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); + writer.forceMerge(1); + writer.close(); + reader = DirectoryReader.open(dir); + doc2 = reader.document(reader.maxDoc() - 1); + fields = doc2.getFields("bin1"); + assertNotNull(fields); + assertEquals(1, fields.length); + b1 = fields[0]; + assertTrue(b1.binaryValue() != null); + bytesRef = b1.binaryValue(); + assertEquals(bin.length, bytesRef.length); + for (int i = 0; i < bin.length; i++) { + assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]); + } + reader.close(); + dir.close(); + } + + /* ??? public void testOpenEmptyDirectory() throws IOException{ + String dirName = "test.empty"; + File fileDirName = new File(dirName); + if (!fileDirName.exists()) { + fileDirName.mkdir(); + } + try { + DirectoryReader.open(fileDirName); + fail("opening DirectoryReader on empty directory failed to produce FileNotFoundException"); + } catch (FileNotFoundException e) { + // GOOD + } + rmDir(fileDirName); + }*/ + +public void testFilesOpenClose() throws IOException { + // Create initial data set + File dirFile = _TestUtil.getTempDir("TestIndexReader.testFilesOpenClose"); + Directory dir = newFSDirectory(dirFile); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + addDoc(writer, "test"); + writer.close(); + dir.close(); + + // Try to erase the data - this ensures that the writer closed all files + _TestUtil.rmDir(dirFile); + dir = newFSDirectory(dirFile); + + // Now create the data set again, just as before + writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); + addDoc(writer, "test"); + writer.close(); + dir.close(); + + // Now open existing directory and test that reader closes all files + dir = newFSDirectory(dirFile); + DirectoryReader reader1 = DirectoryReader.open(dir); + reader1.close(); + dir.close(); + + // The following will fail if reader did not close + // all files + _TestUtil.rmDir(dirFile); + } + + public void testOpenReaderAfterDelete() throws IOException { + File dirFile = _TestUtil.getTempDir("deletetest"); + Directory dir = newFSDirectory(dirFile); + try { + DirectoryReader.open(dir); + fail("expected FileNotFoundException"); + } catch (FileNotFoundException e) { + // expected + } + + dirFile.delete(); + + // Make sure we still get a CorruptIndexException (not NPE): + try { + DirectoryReader.open(dir); + fail("expected FileNotFoundException"); + } catch (FileNotFoundException e) { + // expected + } + + dir.close(); + } + + static void addDocumentWithFields(IndexWriter writer) throws IOException + { + Document doc = new Document(); + + FieldType customType3 = new FieldType(); + customType3.setStored(true); + doc.add(newField("keyword", "test1", StringField.TYPE_STORED)); + doc.add(newField("text", "test1", TextField.TYPE_STORED)); + doc.add(newField("unindexed", "test1", customType3)); + doc.add(new TextField("unstored","test1")); + writer.addDocument(doc); + } + + static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException + { + Document doc = new Document(); + + FieldType customType3 = new FieldType(); + customType3.setStored(true); + doc.add(newField("keyword2", "test1", StringField.TYPE_STORED)); + doc.add(newField("text2", "test1", TextField.TYPE_STORED)); + doc.add(newField("unindexed2", "test1", customType3)); + doc.add(new TextField("unstored2","test1")); + writer.addDocument(doc); + } + + static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException + { + Document doc = new Document(); + FieldType customType5 = new FieldType(TextField.TYPE_STORED); + customType5.setStoreTermVectors(true); + FieldType customType6 = new FieldType(TextField.TYPE_STORED); + customType6.setStoreTermVectors(true); + customType6.setStoreTermVectorOffsets(true); + FieldType customType7 = new FieldType(TextField.TYPE_STORED); + customType7.setStoreTermVectors(true); + customType7.setStoreTermVectorPositions(true); + FieldType customType8 = new FieldType(TextField.TYPE_STORED); + customType8.setStoreTermVectors(true); + customType8.setStoreTermVectorOffsets(true); + customType8.setStoreTermVectorPositions(true); + doc.add(newField("tvnot","tvnot",TextField.TYPE_STORED)); + doc.add(newField("termvector","termvector",customType5)); + doc.add(newField("tvoffset","tvoffset", customType6)); + doc.add(newField("tvposition","tvposition", customType7)); + doc.add(newField("tvpositionoffset","tvpositionoffset", customType8)); + + writer.addDocument(doc); + } + + static void addDoc(IndexWriter writer, String value) throws IOException { + Document doc = new Document(); + doc.add(newField("content", value, TextField.TYPE_UNSTORED)); + writer.addDocument(doc); + } + + // TODO: maybe this can reuse the logic of test dueling codecs? + public static void assertIndexEquals(DirectoryReader index1, DirectoryReader index2) throws IOException { + assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs()); + assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc()); + assertEquals("Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions()); + assertEquals("Single segment test differs.", index1.getSequentialSubReaders().length == 1, index2.getSequentialSubReaders().length == 1); + + // check field names + FieldInfos fieldInfos1 = MultiFields.getMergedFieldInfos(index1); + FieldInfos fieldInfos2 = MultiFields.getMergedFieldInfos(index2); + assertEquals("IndexReaders have different numbers of fields.", fieldInfos1.size(), fieldInfos2.size()); + final int numFields = fieldInfos1.size(); + for(int fieldID=0;fieldID field1 = doc1.getFields(); + List field2 = doc2.getFields(); + assertEquals("Different numbers of fields for doc " + i + ".", field1.size(), field2.size()); + Iterator itField1 = field1.iterator(); + Iterator itField2 = field2.iterator(); + while (itField1.hasNext()) { + Field curField1 = (Field) itField1.next(); + Field curField2 = (Field) itField2.next(); + assertEquals("Different fields names for doc " + i + ".", curField1.name(), curField2.name()); + assertEquals("Different field values for doc " + i + ".", curField1.stringValue(), curField2.stringValue()); + } + } + } + + // check dictionary and posting lists + FieldsEnum fenum1 = MultiFields.getFields(index1).iterator(); + FieldsEnum fenum2 = MultiFields.getFields(index1).iterator(); + String field1 = null; + Bits liveDocs = MultiFields.getLiveDocs(index1); + while((field1=fenum1.next()) != null) { + assertEquals("Different fields", field1, fenum2.next()); + Terms terms1 = fenum1.terms(); + if (terms1 == null) { + assertNull(fenum2.terms()); + continue; + } + TermsEnum enum1 = terms1.iterator(null); + + Terms terms2 = fenum2.terms(); + assertNotNull(terms2); + TermsEnum enum2 = terms2.iterator(null); + + while(enum1.next() != null) { + assertEquals("Different terms", enum1.term(), enum2.next()); + DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null, false); + DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null, false); + + while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + assertEquals("Different doc id in postinglist of term " + enum1.term() + ".", tp1.docID(), tp2.docID()); + assertEquals("Different term frequence in postinglist of term " + enum1.term() + ".", tp1.freq(), tp2.freq()); + for (int i = 0; i < tp1.freq(); i++) { + assertEquals("Different positions in postinglist of term " + enum1.term() + ".", tp1.nextPosition(), tp2.nextPosition()); + } + } + } + } + } + + public void testGetIndexCommit() throws IOException { + + Directory d = newDirectory(); + + // set up writer + IndexWriter writer = new IndexWriter( + d, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDocs(2). + setMergePolicy(newLogMergePolicy(10)) + ); + for(int i=0;i<27;i++) + addDocumentWithFields(writer); + writer.close(); + + SegmentInfos sis = new SegmentInfos(); + sis.read(d); + DirectoryReader r = DirectoryReader.open(d); + IndexCommit c = r.getIndexCommit(); + + assertEquals(sis.getSegmentsFileName(), c.getSegmentsFileName()); + + assertTrue(c.equals(r.getIndexCommit())); + + // Change the index + writer = new IndexWriter( + d, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setOpenMode(OpenMode.APPEND). + setMaxBufferedDocs(2). + setMergePolicy(newLogMergePolicy(10)) + ); + for(int i=0;i<7;i++) + addDocumentWithFields(writer); + writer.close(); + + DirectoryReader r2 = DirectoryReader.openIfChanged(r); + assertNotNull(r2); + assertFalse(c.equals(r2.getIndexCommit())); + assertFalse(r2.getIndexCommit().getSegmentCount() == 1); + r2.close(); + + writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)) + .setOpenMode(OpenMode.APPEND)); + writer.forceMerge(1); + writer.close(); + + r2 = DirectoryReader.openIfChanged(r); + assertNotNull(r2); + assertNull(DirectoryReader.openIfChanged(r2)); + assertEquals(1, r2.getIndexCommit().getSegmentCount()); + + r.close(); + r2.close(); + d.close(); + } + + static Document createDocument(String id) { + Document doc = new Document(); + FieldType customType = new FieldType(TextField.TYPE_STORED); + customType.setTokenized(false); + customType.setOmitNorms(true); + + doc.add(newField("id", id, customType)); + return doc; + } + + // LUCENE-1468 -- make sure on attempting to open an + // DirectoryReader on a non-existent directory, you get a + // good exception + public void testNoDir() throws Throwable { + Directory dir = newFSDirectory(_TestUtil.getTempDir("doesnotexist")); + try { + DirectoryReader.open(dir); + fail("did not hit expected exception"); + } catch (NoSuchDirectoryException nsde) { + // expected + } + dir.close(); + } + + // LUCENE-1509 + public void testNoDupCommitFileNames() throws Throwable { + + Directory dir = newDirectory(); + + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMaxBufferedDocs(2)); + writer.addDocument(createDocument("a")); + writer.addDocument(createDocument("a")); + writer.addDocument(createDocument("a")); + writer.close(); + + Collection commits = DirectoryReader.listCommits(dir); + for (final IndexCommit commit : commits) { + Collection files = commit.getFileNames(); + HashSet seen = new HashSet(); + for (final String fileName : files) { + assertTrue("file " + fileName + " was duplicated", !seen.contains(fileName)); + seen.add(fileName); + } + } + + dir.close(); + } + + // LUCENE-1579: Ensure that on a reopened reader, that any + // shared segments reuse the doc values arrays in + // FieldCache + public void testFieldCacheReuseAfterReopen() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMergePolicy(newLogMergePolicy(10)) + ); + Document doc = new Document(); + doc.add(newField("number", "17", StringField.TYPE_UNSTORED)); + writer.addDocument(doc); + writer.commit(); + + // Open reader1 + DirectoryReader r = DirectoryReader.open(dir); + AtomicReader r1 = getOnlySegmentReader(r); + final int[] ints = FieldCache.DEFAULT.getInts(r1, "number", false); + assertEquals(1, ints.length); + assertEquals(17, ints[0]); + + // Add new segment + writer.addDocument(doc); + writer.commit(); + + // Reopen reader1 --> reader2 + DirectoryReader r2 = DirectoryReader.openIfChanged(r); + assertNotNull(r2); + r.close(); + AtomicReader sub0 = r2.getSequentialSubReaders()[0]; + final int[] ints2 = FieldCache.DEFAULT.getInts(sub0, "number", false); + r2.close(); + assertTrue(ints == ints2); + + writer.close(); + dir.close(); + } + + // LUCENE-1586: getUniqueTermCount + public void testUniqueTermCount() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + Document doc = new Document(); + doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); + doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); + writer.addDocument(doc); + writer.addDocument(doc); + writer.commit(); + + DirectoryReader r = DirectoryReader.open(dir); + AtomicReader r1 = getOnlySegmentReader(r); + assertEquals(36, r1.getUniqueTermCount()); + writer.addDocument(doc); + writer.commit(); + DirectoryReader r2 = DirectoryReader.openIfChanged(r); + assertNotNull(r2); + r.close(); + + IndexReader[] subs = r2.getSequentialSubReaders(); + for(int i=0;i fieldsToLoad = new HashSet(); + assertEquals(0, r.document(0, fieldsToLoad).getFields().size()); + fieldsToLoad.add("field1"); + Document doc2 = r.document(0, fieldsToLoad); + assertEquals(1, doc2.getFields().size()); + assertEquals("foobar", doc2.get("field1")); + r.close(); + dir.close(); + } + +} Index: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (revision 0) +++ lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (working copy) @@ -0,0 +1,639 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +public class TestDirectoryReaderReopen extends LuceneTestCase { + + public void testReopen() throws Exception { + final Directory dir1 = newDirectory(); + + createIndex(random, dir1, false); + performDefaultTests(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestDirectoryReaderReopen.modifyIndex(i, dir1); + } + + @Override + protected DirectoryReader openReader() throws IOException { + return DirectoryReader.open(dir1); + } + + }); + dir1.close(); + + final Directory dir2 = newDirectory(); + + createIndex(random, dir2, true); + performDefaultTests(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestDirectoryReaderReopen.modifyIndex(i, dir2); + } + + @Override + protected DirectoryReader openReader() throws IOException { + return DirectoryReader.open(dir2); + } + + }); + dir2.close(); + } + + // LUCENE-1228: IndexWriter.commit() does not update the index version + // populate an index in iterations. + // at the end of every iteration, commit the index and reopen/recreate the reader. + // in each iteration verify the work of previous iteration. + // try this once with reopen once recreate, on both RAMDir and FSDir. + public void testCommitReopen () throws IOException { + Directory dir = newDirectory(); + doTestReopenWithCommit(random, dir, true); + dir.close(); + } + public void testCommitRecreate () throws IOException { + Directory dir = newDirectory(); + doTestReopenWithCommit(random, dir, false); + dir.close(); + } + + private void doTestReopenWithCommit (Random random, Directory dir, boolean withReopen) throws IOException { + IndexWriter iwriter = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( + OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy())); + iwriter.commit(); + DirectoryReader reader = DirectoryReader.open(dir); + try { + int M = 3; + FieldType customType = new FieldType(TextField.TYPE_STORED); + customType.setTokenized(false); + FieldType customType2 = new FieldType(TextField.TYPE_STORED); + customType2.setTokenized(false); + customType2.setOmitNorms(true); + FieldType customType3 = new FieldType(); + customType3.setStored(true); + for (int i=0; i<4; i++) { + for (int j=0; j0) { + int k = i-1; + int n = j + k*M; + Document prevItereationDoc = reader.document(n); + assertNotNull(prevItereationDoc); + String id = prevItereationDoc.get("id"); + assertEquals(k+"_"+j, id); + } + } + iwriter.commit(); + if (withReopen) { + // reopen + DirectoryReader r2 = DirectoryReader.openIfChanged(reader); + if (r2 != null) { + reader.close(); + reader = r2; + } + } else { + // recreate + reader.close(); + reader = DirectoryReader.open(dir); + } + } + } finally { + iwriter.close(); + reader.close(); + } + } + + private void performDefaultTests(TestReopen test) throws Exception { + + DirectoryReader index1 = test.openReader(); + DirectoryReader index2 = test.openReader(); + + TestDirectoryReader.assertIndexEquals(index1, index2); + + // verify that reopen() does not return a new reader instance + // in case the index has no changes + ReaderCouple couple = refreshReader(index2, false); + assertTrue(couple.refreshedReader == index2); + + couple = refreshReader(index2, test, 0, true); + index1.close(); + index1 = couple.newReader; + + DirectoryReader index2_refreshed = couple.refreshedReader; + index2.close(); + + // test if refreshed reader and newly opened reader return equal results + TestDirectoryReader.assertIndexEquals(index1, index2_refreshed); + + index2_refreshed.close(); + assertReaderClosed(index2, true, true); + assertReaderClosed(index2_refreshed, true, true); + + index2 = test.openReader(); + + for (int i = 1; i < 4; i++) { + + index1.close(); + couple = refreshReader(index2, test, i, true); + // refresh DirectoryReader + index2.close(); + + index2 = couple.refreshedReader; + index1 = couple.newReader; + TestDirectoryReader.assertIndexEquals(index1, index2); + } + + index1.close(); + index2.close(); + assertReaderClosed(index1, true, true); + assertReaderClosed(index2, true, true); + } + + private void performTestsWithExceptionInReopen(TestReopen test) throws Exception { + DirectoryReader index1 = test.openReader(); + DirectoryReader index2 = test.openReader(); + + TestDirectoryReader.assertIndexEquals(index1, index2); + + try { + refreshReader(index1, test, 0, true); + fail("Expected exception not thrown."); + } catch (Exception e) { + // expected exception + } + + // index2 should still be usable and unaffected by the failed reopen() call + TestDirectoryReader.assertIndexEquals(index1, index2); + + index1.close(); + index2.close(); + } + + public void testThreadSafety() throws Exception { + final Directory dir = newDirectory(); + // NOTE: this also controls the number of threads! + final int n = _TestUtil.nextInt(random, 20, 40); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random))); + for (int i = 0; i < n; i++) { + writer.addDocument(createDocument(i, 3)); + } + writer.forceMerge(1); + writer.close(); + + final TestReopen test = new TestReopen() { + @Override + protected void modifyIndex(int i) throws IOException { + IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random))); + modifier.addDocument(createDocument(n + i, 6)); + modifier.close(); + } + + @Override + protected DirectoryReader openReader() throws IOException { + return DirectoryReader.open(dir); + } + }; + + final List readers = Collections.synchronizedList(new ArrayList()); + DirectoryReader firstReader = DirectoryReader.open(dir); + DirectoryReader reader = firstReader; + final Random rnd = random; + + ReaderThread[] threads = new ReaderThread[n]; + final Set readersToClose = Collections.synchronizedSet(new HashSet()); + + for (int i = 0; i < n; i++) { + if (i % 2 == 0) { + DirectoryReader refreshed = DirectoryReader.openIfChanged(reader); + if (refreshed != null) { + readersToClose.add(reader); + reader = refreshed; + } + } + final DirectoryReader r = reader; + + final int index = i; + + ReaderThreadTask task; + + if (i < 4 || (i >=10 && i < 14) || i > 18) { + task = new ReaderThreadTask() { + + @Override + public void run() throws Exception { + while (!stopped) { + if (index % 2 == 0) { + // refresh reader synchronized + ReaderCouple c = (refreshReader(r, test, index, true)); + readersToClose.add(c.newReader); + readersToClose.add(c.refreshedReader); + readers.add(c); + // prevent too many readers + break; + } else { + // not synchronized + DirectoryReader refreshed = DirectoryReader.openIfChanged(r); + if (refreshed == null) { + refreshed = r; + } + + IndexSearcher searcher = newSearcher(refreshed); + ScoreDoc[] hits = searcher.search( + new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))), + null, 1000).scoreDocs; + if (hits.length > 0) { + searcher.doc(hits[0].doc); + } + if (refreshed != r) { + refreshed.close(); + } + } + synchronized(this) { + wait(_TestUtil.nextInt(random, 1, 100)); + } + } + } + + }; + } else { + task = new ReaderThreadTask() { + @Override + public void run() throws Exception { + while (!stopped) { + int numReaders = readers.size(); + if (numReaders > 0) { + ReaderCouple c = readers.get(rnd.nextInt(numReaders)); + TestDirectoryReader.assertIndexEquals(c.newReader, c.refreshedReader); + } + + synchronized(this) { + wait(_TestUtil.nextInt(random, 1, 100)); + } + } + } + }; + } + + threads[i] = new ReaderThread(task); + threads[i].start(); + } + + synchronized(this) { + wait(1000); + } + + for (int i = 0; i < n; i++) { + if (threads[i] != null) { + threads[i].stopThread(); + } + } + + for (int i = 0; i < n; i++) { + if (threads[i] != null) { + threads[i].join(); + if (threads[i].error != null) { + String msg = "Error occurred in thread " + threads[i].getName() + ":\n" + threads[i].error.getMessage(); + fail(msg); + } + } + + } + + for (final DirectoryReader readerToClose : readersToClose) { + readerToClose.close(); + } + + firstReader.close(); + reader.close(); + + for (final DirectoryReader readerToClose : readersToClose) { + assertReaderClosed(readerToClose, true, true); + } + + assertReaderClosed(reader, true, true); + assertReaderClosed(firstReader, true, true); + + dir.close(); + } + + private static class ReaderCouple { + ReaderCouple(DirectoryReader r1, DirectoryReader r2) { + newReader = r1; + refreshedReader = r2; + } + + DirectoryReader newReader; + DirectoryReader refreshedReader; + } + + private abstract static class ReaderThreadTask { + protected volatile boolean stopped; + public void stop() { + this.stopped = true; + } + + public abstract void run() throws Exception; + } + + private static class ReaderThread extends Thread { + private ReaderThreadTask task; + private Throwable error; + + + ReaderThread(ReaderThreadTask task) { + this.task = task; + } + + public void stopThread() { + this.task.stop(); + } + + @Override + public void run() { + try { + this.task.run(); + } catch (Throwable r) { + r.printStackTrace(System.out); + this.error = r; + } + } + } + + private Object createReaderMutex = new Object(); + + private ReaderCouple refreshReader(DirectoryReader reader, boolean hasChanges) throws IOException { + return refreshReader(reader, null, -1, hasChanges); + } + + ReaderCouple refreshReader(DirectoryReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException { + synchronized (createReaderMutex) { + DirectoryReader r = null; + if (test != null) { + test.modifyIndex(modify); + r = test.openReader(); + } + + DirectoryReader refreshed = null; + try { + refreshed = DirectoryReader.openIfChanged(reader); + if (refreshed == null) { + refreshed = reader; + } + } finally { + if (refreshed == null && r != null) { + // Hit exception -- close opened reader + r.close(); + } + } + + if (hasChanges) { + if (refreshed == reader) { + fail("No new DirectoryReader instance created during refresh."); + } + } else { + if (refreshed != reader) { + fail("New DirectoryReader instance created during refresh even though index had no changes."); + } + } + + return new ReaderCouple(r, refreshed); + } + } + + public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException { + IndexWriter.unlock(dir); + IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random, + TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(new LogDocMergePolicy())); + + for (int i = 0; i < 100; i++) { + w.addDocument(createDocument(i, 4)); + if (multiSegment && (i % 10) == 0) { + w.commit(); + } + } + + if (!multiSegment) { + w.forceMerge(1); + } + + w.close(); + + DirectoryReader r = DirectoryReader.open(dir); + if (multiSegment) { + assertTrue(r.getSequentialSubReaders().length > 1); + } else { + assertTrue(r.getSequentialSubReaders().length == 1); + } + r.close(); + } + + public static Document createDocument(int n, int numFields) { + StringBuilder sb = new StringBuilder(); + Document doc = new Document(); + sb.append("a"); + sb.append(n); + FieldType customType2 = new FieldType(TextField.TYPE_STORED); + customType2.setTokenized(false); + customType2.setOmitNorms(true); + FieldType customType3 = new FieldType(); + customType3.setStored(true); + doc.add(new Field("field1", sb.toString(), TextField.TYPE_STORED)); + doc.add(new Field("fielda", sb.toString(), customType2)); + doc.add(new Field("fieldb", sb.toString(), customType3)); + sb.append(" b"); + sb.append(n); + for (int i = 1; i < numFields; i++) { + doc.add(new Field("field" + (i+1), sb.toString(), TextField.TYPE_STORED)); + } + return doc; + } + + static void modifyIndex(int i, Directory dir) throws IOException { + switch (i) { + case 0: { + if (VERBOSE) { + System.out.println("TEST: modify index"); + } + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.deleteDocuments(new Term("field2", "a11")); + w.deleteDocuments(new Term("field2", "b30")); + w.close(); + break; + } + case 1: { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.forceMerge(1); + w.close(); + break; + } + case 2: { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.addDocument(createDocument(101, 4)); + w.forceMerge(1); + w.addDocument(createDocument(102, 4)); + w.addDocument(createDocument(103, 4)); + w.close(); + break; + } + case 3: { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.addDocument(createDocument(101, 4)); + w.close(); + break; + } + } + } + + static void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) { + assertEquals(0, reader.getRefCount()); + + if (checkNormsClosed && reader instanceof AtomicReader) { + // TODO: should we really assert something here? we check for open files and this is obselete... + // assertTrue(((SegmentReader) reader).normsClosed()); + } + + if (checkSubReaders && reader instanceof CompositeReader) { + IndexReader[] subReaders = ((CompositeReader) reader).getSequentialSubReaders(); + for (int i = 0; i < subReaders.length; i++) { + assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); + } + } + } + + /* + private void assertReaderOpen(DirectoryReader reader) { + reader.ensureOpen(); + + if (reader instanceof DirectoryReader) { + DirectoryReader[] subReaders = reader.getSequentialSubReaders(); + for (int i = 0; i < subReaders.length; i++) { + assertReaderOpen(subReaders[i]); + } + } + } + */ + + private void assertRefCountEquals(int refCount, DirectoryReader reader) { + assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount()); + } + + + private abstract static class TestReopen { + protected abstract DirectoryReader openReader() throws IOException; + protected abstract void modifyIndex(int i) throws IOException; + } + + private static class KeepAllCommits implements IndexDeletionPolicy { + public void onInit(List commits) { + } + public void onCommit(List commits) { + } + } + + public void testReopenOnCommit() throws Throwable { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setIndexDeletionPolicy(new KeepAllCommits()). + setMaxBufferedDocs(-1). + setMergePolicy(newLogMergePolicy(10)) + ); + for(int i=0;i<4;i++) { + Document doc = new Document(); + doc.add(newField("id", ""+i, StringField.TYPE_UNSTORED)); + writer.addDocument(doc); + Map data = new HashMap(); + data.put("index", i+""); + writer.commit(data); + } + for(int i=0;i<4;i++) { + writer.deleteDocuments(new Term("id", ""+i)); + Map data = new HashMap(); + data.put("index", (4+i)+""); + writer.commit(data); + } + writer.close(); + + DirectoryReader r = DirectoryReader.open(dir); + assertEquals(0, r.numDocs()); + + Collection commits = DirectoryReader.listCommits(dir); + for (final IndexCommit commit : commits) { + DirectoryReader r2 = DirectoryReader.openIfChanged(r, commit); + assertNotNull(r2); + assertTrue(r2 != r); + + final Map s = commit.getUserData(); + final int v; + if (s.size() == 0) { + // First commit created by IW + v = -1; + } else { + v = Integer.parseInt(s.get("index")); + } + if (v < 4) { + assertEquals(1+v, r2.numDocs()); + } else { + assertEquals(7-v, r2.numDocs()); + } + r.close(); + r = r2; + } + r.close(); + dir.close(); + } +} Index: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (revision 1243051) +++ lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (working copy) Property changes on: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/core/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexReader.java (revision 1243051) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -1,1004 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldCache; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.store.NoSuchDirectoryException; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.ReaderUtil; -import org.apache.lucene.util._TestUtil; -import org.junit.Assume; - -public class TestIndexReader extends LuceneTestCase { - - public void testIsCurrent() throws Exception { - Directory d = newDirectory(); - IndexWriter writer = new IndexWriter(d, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - writer.close(); - // set up reader: - DirectoryReader reader = DirectoryReader.open(d); - assertTrue(reader.isCurrent()); - // modify index by adding another document: - writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); - addDocumentWithFields(writer); - writer.close(); - assertFalse(reader.isCurrent()); - // re-create index: - writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); - addDocumentWithFields(writer); - writer.close(); - assertFalse(reader.isCurrent()); - reader.close(); - d.close(); - } - - /** - * Tests the IndexReader.getFieldNames implementation - * @throws Exception on error - */ - public void testGetFieldNames() throws Exception { - Directory d = newDirectory(); - // set up writer - IndexWriter writer = new IndexWriter( - d, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) - ); - - Document doc = new Document(); - - FieldType customType3 = new FieldType(); - customType3.setStored(true); - - doc.add(new Field("keyword", "test1", StringField.TYPE_STORED)); - doc.add(new Field("text", "test1", TextField.TYPE_STORED)); - doc.add(new Field("unindexed", "test1", customType3)); - doc.add(new TextField("unstored","test1")); - writer.addDocument(doc); - - writer.close(); - // set up reader - DirectoryReader reader = DirectoryReader.open(d); - FieldInfos fieldInfos = MultiFields.getMergedFieldInfos(reader); - assertNotNull(fieldInfos.fieldInfo("keyword")); - assertNotNull(fieldInfos.fieldInfo("text")); - assertNotNull(fieldInfos.fieldInfo("unindexed")); - assertNotNull(fieldInfos.fieldInfo("unstored")); - reader.close(); - // add more documents - writer = new IndexWriter( - d, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setOpenMode(OpenMode.APPEND). - setMergePolicy(newLogMergePolicy()) - ); - // want to get some more segments here - int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor(); - for (int i = 0; i < 5*mergeFactor; i++) { - doc = new Document(); - doc.add(new Field("keyword", "test1", StringField.TYPE_STORED)); - doc.add(new Field("text", "test1", TextField.TYPE_STORED)); - doc.add(new Field("unindexed", "test1", customType3)); - doc.add(new TextField("unstored","test1")); - writer.addDocument(doc); - } - // new fields are in some different segments (we hope) - for (int i = 0; i < 5*mergeFactor; i++) { - doc = new Document(); - doc.add(new Field("keyword2", "test1", StringField.TYPE_STORED)); - doc.add(new Field("text2", "test1", TextField.TYPE_STORED)); - doc.add(new Field("unindexed2", "test1", customType3)); - doc.add(new TextField("unstored2","test1")); - writer.addDocument(doc); - } - // new termvector fields - - FieldType customType5 = new FieldType(TextField.TYPE_STORED); - customType5.setStoreTermVectors(true); - FieldType customType6 = new FieldType(TextField.TYPE_STORED); - customType6.setStoreTermVectors(true); - customType6.setStoreTermVectorOffsets(true); - FieldType customType7 = new FieldType(TextField.TYPE_STORED); - customType7.setStoreTermVectors(true); - customType7.setStoreTermVectorPositions(true); - FieldType customType8 = new FieldType(TextField.TYPE_STORED); - customType8.setStoreTermVectors(true); - customType8.setStoreTermVectorOffsets(true); - customType8.setStoreTermVectorPositions(true); - - for (int i = 0; i < 5*mergeFactor; i++) { - doc = new Document(); - doc.add(new Field("tvnot", "tvnot", TextField.TYPE_STORED)); - doc.add(new Field("termvector", "termvector", customType5)); - doc.add(new Field("tvoffset", "tvoffset", customType6)); - doc.add(new Field("tvposition", "tvposition", customType7)); - doc.add(new Field("tvpositionoffset", "tvpositionoffset", customType8)); - writer.addDocument(doc); - } - - writer.close(); - - // verify fields again - reader = DirectoryReader.open(d); - fieldInfos = MultiFields.getMergedFieldInfos(reader); - - Collection allFieldNames = new HashSet(); - Collection indexedFieldNames = new HashSet(); - Collection notIndexedFieldNames = new HashSet(); - Collection tvFieldNames = new HashSet(); - - for(FieldInfo fieldInfo : fieldInfos) { - final String name = fieldInfo.name; - allFieldNames.add(name); - if (fieldInfo.isIndexed) { - indexedFieldNames.add(name); - } else { - notIndexedFieldNames.add(name); - } - if (fieldInfo.storeTermVector) { - tvFieldNames.add(name); - } - } - - assertTrue(allFieldNames.contains("keyword")); - assertTrue(allFieldNames.contains("text")); - assertTrue(allFieldNames.contains("unindexed")); - assertTrue(allFieldNames.contains("unstored")); - assertTrue(allFieldNames.contains("keyword2")); - assertTrue(allFieldNames.contains("text2")); - assertTrue(allFieldNames.contains("unindexed2")); - assertTrue(allFieldNames.contains("unstored2")); - assertTrue(allFieldNames.contains("tvnot")); - assertTrue(allFieldNames.contains("termvector")); - assertTrue(allFieldNames.contains("tvposition")); - assertTrue(allFieldNames.contains("tvoffset")); - assertTrue(allFieldNames.contains("tvpositionoffset")); - - // verify that only indexed fields were returned - assertEquals(11, indexedFieldNames.size()); // 6 original + the 5 termvector fields - assertTrue(indexedFieldNames.contains("keyword")); - assertTrue(indexedFieldNames.contains("text")); - assertTrue(indexedFieldNames.contains("unstored")); - assertTrue(indexedFieldNames.contains("keyword2")); - assertTrue(indexedFieldNames.contains("text2")); - assertTrue(indexedFieldNames.contains("unstored2")); - assertTrue(indexedFieldNames.contains("tvnot")); - assertTrue(indexedFieldNames.contains("termvector")); - assertTrue(indexedFieldNames.contains("tvposition")); - assertTrue(indexedFieldNames.contains("tvoffset")); - assertTrue(indexedFieldNames.contains("tvpositionoffset")); - - // verify that only unindexed fields were returned - assertEquals(2, notIndexedFieldNames.size()); // the following fields - assertTrue(notIndexedFieldNames.contains("unindexed")); - assertTrue(notIndexedFieldNames.contains("unindexed2")); - - // verify index term vector fields - assertEquals(tvFieldNames.toString(), 4, tvFieldNames.size()); // 4 field has term vector only - assertTrue(tvFieldNames.contains("termvector")); - - reader.close(); - d.close(); - } - - public void testTermVectors() throws Exception { - Directory d = newDirectory(); - // set up writer - IndexWriter writer = new IndexWriter( - d, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setMergePolicy(newLogMergePolicy()) - ); - // want to get some more segments here - // new termvector fields - int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor(); - FieldType customType5 = new FieldType(TextField.TYPE_STORED); - customType5.setStoreTermVectors(true); - FieldType customType6 = new FieldType(TextField.TYPE_STORED); - customType6.setStoreTermVectors(true); - customType6.setStoreTermVectorOffsets(true); - FieldType customType7 = new FieldType(TextField.TYPE_STORED); - customType7.setStoreTermVectors(true); - customType7.setStoreTermVectorPositions(true); - FieldType customType8 = new FieldType(TextField.TYPE_STORED); - customType8.setStoreTermVectors(true); - customType8.setStoreTermVectorOffsets(true); - customType8.setStoreTermVectorPositions(true); - for (int i = 0; i < 5 * mergeFactor; i++) { - Document doc = new Document(); - doc.add(new Field("tvnot", "one two two three three three", TextField.TYPE_STORED)); - doc.add(new Field("termvector", "one two two three three three", customType5)); - doc.add(new Field("tvoffset", "one two two three three three", customType6)); - doc.add(new Field("tvposition", "one two two three three three", customType7)); - doc.add(new Field("tvpositionoffset", "one two two three three three", customType8)); - - writer.addDocument(doc); - } - writer.close(); - d.close(); - } - - static void assertTermDocsCount(String msg, - IndexReader reader, - Term term, - int expected) - throws IOException { - DocsEnum tdocs = _TestUtil.docs(random, reader, - term.field(), - new BytesRef(term.text()), - MultiFields.getLiveDocs(reader), - null, - false); - int count = 0; - if (tdocs != null) { - while(tdocs.nextDoc()!= DocIdSetIterator.NO_MORE_DOCS) { - count++; - } - } - assertEquals(msg + ", count mismatch", expected, count); - } - - - public void testBinaryFields() throws IOException { - Directory dir = newDirectory(); - byte[] bin = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - - for (int i = 0; i < 10; i++) { - addDoc(writer, "document number " + (i + 1)); - addDocumentWithFields(writer); - addDocumentWithDifferentFields(writer); - addDocumentWithTermVectorFields(writer); - } - writer.close(); - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); - Document doc = new Document(); - doc.add(new StoredField("bin1", bin)); - doc.add(new TextField("junk", "junk text")); - writer.addDocument(doc); - writer.close(); - DirectoryReader reader = DirectoryReader.open(dir); - Document doc2 = reader.document(reader.maxDoc() - 1); - IndexableField[] fields = doc2.getFields("bin1"); - assertNotNull(fields); - assertEquals(1, fields.length); - IndexableField b1 = fields[0]; - assertTrue(b1.binaryValue() != null); - BytesRef bytesRef = b1.binaryValue(); - assertEquals(bin.length, bytesRef.length); - for (int i = 0; i < bin.length; i++) { - assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]); - } - reader.close(); - // force merge - - - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); - writer.forceMerge(1); - writer.close(); - reader = DirectoryReader.open(dir); - doc2 = reader.document(reader.maxDoc() - 1); - fields = doc2.getFields("bin1"); - assertNotNull(fields); - assertEquals(1, fields.length); - b1 = fields[0]; - assertTrue(b1.binaryValue() != null); - bytesRef = b1.binaryValue(); - assertEquals(bin.length, bytesRef.length); - for (int i = 0; i < bin.length; i++) { - assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]); - } - reader.close(); - dir.close(); - } - - /* ??? public void testOpenEmptyDirectory() throws IOException{ - String dirName = "test.empty"; - File fileDirName = new File(dirName); - if (!fileDirName.exists()) { - fileDirName.mkdir(); - } - try { - DirectoryReader.open(fileDirName); - fail("opening DirectoryReader on empty directory failed to produce FileNotFoundException"); - } catch (FileNotFoundException e) { - // GOOD - } - rmDir(fileDirName); - }*/ - - public void testFilesOpenClose() throws IOException { - // Create initial data set - File dirFile = _TestUtil.getTempDir("TestIndexReader.testFilesOpenClose"); - Directory dir = newFSDirectory(dirFile); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDoc(writer, "test"); - writer.close(); - dir.close(); - - // Try to erase the data - this ensures that the writer closed all files - _TestUtil.rmDir(dirFile); - dir = newFSDirectory(dirFile); - - // Now create the data set again, just as before - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); - addDoc(writer, "test"); - writer.close(); - dir.close(); - - // Now open existing directory and test that reader closes all files - dir = newFSDirectory(dirFile); - DirectoryReader reader1 = DirectoryReader.open(dir); - reader1.close(); - dir.close(); - - // The following will fail if reader did not close - // all files - _TestUtil.rmDir(dirFile); - } - - public void testOpenReaderAfterDelete() throws IOException { - File dirFile = _TestUtil.getTempDir("deletetest"); - Directory dir = newFSDirectory(dirFile); - try { - DirectoryReader.open(dir); - fail("expected FileNotFoundException"); - } catch (FileNotFoundException e) { - // expected - } - - dirFile.delete(); - - // Make sure we still get a CorruptIndexException (not NPE): - try { - DirectoryReader.open(dir); - fail("expected FileNotFoundException"); - } catch (FileNotFoundException e) { - // expected - } - - dir.close(); - } - - static void addDocumentWithFields(IndexWriter writer) throws IOException - { - Document doc = new Document(); - - FieldType customType3 = new FieldType(); - customType3.setStored(true); - doc.add(newField("keyword", "test1", StringField.TYPE_STORED)); - doc.add(newField("text", "test1", TextField.TYPE_STORED)); - doc.add(newField("unindexed", "test1", customType3)); - doc.add(new TextField("unstored","test1")); - writer.addDocument(doc); - } - - static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException - { - Document doc = new Document(); - - FieldType customType3 = new FieldType(); - customType3.setStored(true); - doc.add(newField("keyword2", "test1", StringField.TYPE_STORED)); - doc.add(newField("text2", "test1", TextField.TYPE_STORED)); - doc.add(newField("unindexed2", "test1", customType3)); - doc.add(new TextField("unstored2","test1")); - writer.addDocument(doc); - } - - static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException - { - Document doc = new Document(); - FieldType customType5 = new FieldType(TextField.TYPE_STORED); - customType5.setStoreTermVectors(true); - FieldType customType6 = new FieldType(TextField.TYPE_STORED); - customType6.setStoreTermVectors(true); - customType6.setStoreTermVectorOffsets(true); - FieldType customType7 = new FieldType(TextField.TYPE_STORED); - customType7.setStoreTermVectors(true); - customType7.setStoreTermVectorPositions(true); - FieldType customType8 = new FieldType(TextField.TYPE_STORED); - customType8.setStoreTermVectors(true); - customType8.setStoreTermVectorOffsets(true); - customType8.setStoreTermVectorPositions(true); - doc.add(newField("tvnot","tvnot",TextField.TYPE_STORED)); - doc.add(newField("termvector","termvector",customType5)); - doc.add(newField("tvoffset","tvoffset", customType6)); - doc.add(newField("tvposition","tvposition", customType7)); - doc.add(newField("tvpositionoffset","tvpositionoffset", customType8)); - - writer.addDocument(doc); - } - - static void addDoc(IndexWriter writer, String value) throws IOException { - Document doc = new Document(); - doc.add(newField("content", value, TextField.TYPE_UNSTORED)); - writer.addDocument(doc); - } - - // TODO: maybe this can reuse the logic of test dueling codecs? - public static void assertIndexEquals(DirectoryReader index1, DirectoryReader index2) throws IOException { - assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs()); - assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc()); - assertEquals("Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions()); - assertEquals("Single segment test differs.", index1.getSequentialSubReaders().length == 1, index2.getSequentialSubReaders().length == 1); - - // check field names - FieldInfos fieldInfos1 = MultiFields.getMergedFieldInfos(index1); - FieldInfos fieldInfos2 = MultiFields.getMergedFieldInfos(index2); - assertEquals("IndexReaders have different numbers of fields.", fieldInfos1.size(), fieldInfos2.size()); - final int numFields = fieldInfos1.size(); - for(int fieldID=0;fieldID field1 = doc1.getFields(); - List field2 = doc2.getFields(); - assertEquals("Different numbers of fields for doc " + i + ".", field1.size(), field2.size()); - Iterator itField1 = field1.iterator(); - Iterator itField2 = field2.iterator(); - while (itField1.hasNext()) { - Field curField1 = (Field) itField1.next(); - Field curField2 = (Field) itField2.next(); - assertEquals("Different fields names for doc " + i + ".", curField1.name(), curField2.name()); - assertEquals("Different field values for doc " + i + ".", curField1.stringValue(), curField2.stringValue()); - } - } - } - - // check dictionary and posting lists - FieldsEnum fenum1 = MultiFields.getFields(index1).iterator(); - FieldsEnum fenum2 = MultiFields.getFields(index1).iterator(); - String field1 = null; - Bits liveDocs = MultiFields.getLiveDocs(index1); - while((field1=fenum1.next()) != null) { - assertEquals("Different fields", field1, fenum2.next()); - Terms terms1 = fenum1.terms(); - if (terms1 == null) { - assertNull(fenum2.terms()); - continue; - } - TermsEnum enum1 = terms1.iterator(null); - - Terms terms2 = fenum2.terms(); - assertNotNull(terms2); - TermsEnum enum2 = terms2.iterator(null); - - while(enum1.next() != null) { - assertEquals("Different terms", enum1.term(), enum2.next()); - DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null, false); - DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null, false); - - while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { - assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertEquals("Different doc id in postinglist of term " + enum1.term() + ".", tp1.docID(), tp2.docID()); - assertEquals("Different term frequence in postinglist of term " + enum1.term() + ".", tp1.freq(), tp2.freq()); - for (int i = 0; i < tp1.freq(); i++) { - assertEquals("Different positions in postinglist of term " + enum1.term() + ".", tp1.nextPosition(), tp2.nextPosition()); - } - } - } - } - } - - public void testGetIndexCommit() throws IOException { - - Directory d = newDirectory(); - - // set up writer - IndexWriter writer = new IndexWriter( - d, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setMaxBufferedDocs(2). - setMergePolicy(newLogMergePolicy(10)) - ); - for(int i=0;i<27;i++) - addDocumentWithFields(writer); - writer.close(); - - SegmentInfos sis = new SegmentInfos(); - sis.read(d); - DirectoryReader r = DirectoryReader.open(d); - IndexCommit c = r.getIndexCommit(); - - assertEquals(sis.getSegmentsFileName(), c.getSegmentsFileName()); - - assertTrue(c.equals(r.getIndexCommit())); - - // Change the index - writer = new IndexWriter( - d, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setOpenMode(OpenMode.APPEND). - setMaxBufferedDocs(2). - setMergePolicy(newLogMergePolicy(10)) - ); - for(int i=0;i<7;i++) - addDocumentWithFields(writer); - writer.close(); - - DirectoryReader r2 = DirectoryReader.openIfChanged(r); - assertNotNull(r2); - assertFalse(c.equals(r2.getIndexCommit())); - assertFalse(r2.getIndexCommit().getSegmentCount() == 1); - r2.close(); - - writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)) - .setOpenMode(OpenMode.APPEND)); - writer.forceMerge(1); - writer.close(); - - r2 = DirectoryReader.openIfChanged(r); - assertNotNull(r2); - assertNull(DirectoryReader.openIfChanged(r2)); - assertEquals(1, r2.getIndexCommit().getSegmentCount()); - - r.close(); - r2.close(); - d.close(); - } - - static Document createDocument(String id) { - Document doc = new Document(); - FieldType customType = new FieldType(TextField.TYPE_STORED); - customType.setTokenized(false); - customType.setOmitNorms(true); - - doc.add(newField("id", id, customType)); - return doc; - } - - // LUCENE-1468 -- make sure on attempting to open an - // DirectoryReader on a non-existent directory, you get a - // good exception - public void testNoDir() throws Throwable { - Directory dir = newFSDirectory(_TestUtil.getTempDir("doesnotexist")); - try { - DirectoryReader.open(dir); - fail("did not hit expected exception"); - } catch (NoSuchDirectoryException nsde) { - // expected - } - dir.close(); - } - - // LUCENE-1509 - public void testNoDupCommitFileNames() throws Throwable { - - Directory dir = newDirectory(); - - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setMaxBufferedDocs(2)); - writer.addDocument(createDocument("a")); - writer.addDocument(createDocument("a")); - writer.addDocument(createDocument("a")); - writer.close(); - - Collection commits = DirectoryReader.listCommits(dir); - for (final IndexCommit commit : commits) { - Collection files = commit.getFileNames(); - HashSet seen = new HashSet(); - for (final String fileName : files) { - assertTrue("file " + fileName + " was duplicated", !seen.contains(fileName)); - seen.add(fileName); - } - } - - dir.close(); - } - - // LUCENE-1579: Ensure that on a reopened reader, that any - // shared segments reuse the doc values arrays in - // FieldCache - public void testFieldCacheReuseAfterReopen() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter( - dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setMergePolicy(newLogMergePolicy(10)) - ); - Document doc = new Document(); - doc.add(newField("number", "17", StringField.TYPE_UNSTORED)); - writer.addDocument(doc); - writer.commit(); - - // Open reader1 - DirectoryReader r = DirectoryReader.open(dir); - AtomicReader r1 = getOnlySegmentReader(r); - final int[] ints = FieldCache.DEFAULT.getInts(r1, "number", false); - assertEquals(1, ints.length); - assertEquals(17, ints[0]); - - // Add new segment - writer.addDocument(doc); - writer.commit(); - - // Reopen reader1 --> reader2 - DirectoryReader r2 = DirectoryReader.openIfChanged(r); - assertNotNull(r2); - r.close(); - AtomicReader sub0 = r2.getSequentialSubReaders()[0]; - final int[] ints2 = FieldCache.DEFAULT.getInts(sub0, "number", false); - r2.close(); - assertTrue(ints == ints2); - - writer.close(); - dir.close(); - } - - // LUCENE-1586: getUniqueTermCount - public void testUniqueTermCount() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - Document doc = new Document(); - doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); - doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); - writer.addDocument(doc); - writer.addDocument(doc); - writer.commit(); - - DirectoryReader r = DirectoryReader.open(dir); - AtomicReader r1 = getOnlySegmentReader(r); - assertEquals(36, r1.getUniqueTermCount()); - writer.addDocument(doc); - writer.commit(); - DirectoryReader r2 = DirectoryReader.openIfChanged(r); - assertNotNull(r2); - r.close(); - - IndexReader[] subs = r2.getSequentialSubReaders(); - for(int i=0;i fieldsToLoad = new HashSet(); - assertEquals(0, r.document(0, fieldsToLoad).getFields().size()); - fieldsToLoad.add("field1"); - Document doc2 = r.document(0, fieldsToLoad); - assertEquals(1, doc2.getFields().size()); - assertEquals("foobar", doc2.get("field1")); - r.close(); - dir.close(); - } -} Index: lucene/core/src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 1243051) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -1,639 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.FieldCache; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; - -public class TestIndexReaderReopen extends LuceneTestCase { - - public void testReopen() throws Exception { - final Directory dir1 = newDirectory(); - - createIndex(random, dir1, false); - performDefaultTests(new TestReopen() { - - @Override - protected void modifyIndex(int i) throws IOException { - TestIndexReaderReopen.modifyIndex(i, dir1); - } - - @Override - protected DirectoryReader openReader() throws IOException { - return DirectoryReader.open(dir1); - } - - }); - dir1.close(); - - final Directory dir2 = newDirectory(); - - createIndex(random, dir2, true); - performDefaultTests(new TestReopen() { - - @Override - protected void modifyIndex(int i) throws IOException { - TestIndexReaderReopen.modifyIndex(i, dir2); - } - - @Override - protected DirectoryReader openReader() throws IOException { - return DirectoryReader.open(dir2); - } - - }); - dir2.close(); - } - - // LUCENE-1228: IndexWriter.commit() does not update the index version - // populate an index in iterations. - // at the end of every iteration, commit the index and reopen/recreate the reader. - // in each iteration verify the work of previous iteration. - // try this once with reopen once recreate, on both RAMDir and FSDir. - public void testCommitReopen () throws IOException { - Directory dir = newDirectory(); - doTestReopenWithCommit(random, dir, true); - dir.close(); - } - public void testCommitRecreate () throws IOException { - Directory dir = newDirectory(); - doTestReopenWithCommit(random, dir, false); - dir.close(); - } - - private void doTestReopenWithCommit (Random random, Directory dir, boolean withReopen) throws IOException { - IndexWriter iwriter = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( - OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy())); - iwriter.commit(); - DirectoryReader reader = DirectoryReader.open(dir); - try { - int M = 3; - FieldType customType = new FieldType(TextField.TYPE_STORED); - customType.setTokenized(false); - FieldType customType2 = new FieldType(TextField.TYPE_STORED); - customType2.setTokenized(false); - customType2.setOmitNorms(true); - FieldType customType3 = new FieldType(); - customType3.setStored(true); - for (int i=0; i<4; i++) { - for (int j=0; j0) { - int k = i-1; - int n = j + k*M; - Document prevItereationDoc = reader.document(n); - assertNotNull(prevItereationDoc); - String id = prevItereationDoc.get("id"); - assertEquals(k+"_"+j, id); - } - } - iwriter.commit(); - if (withReopen) { - // reopen - DirectoryReader r2 = DirectoryReader.openIfChanged(reader); - if (r2 != null) { - reader.close(); - reader = r2; - } - } else { - // recreate - reader.close(); - reader = DirectoryReader.open(dir); - } - } - } finally { - iwriter.close(); - reader.close(); - } - } - - private void performDefaultTests(TestReopen test) throws Exception { - - DirectoryReader index1 = test.openReader(); - DirectoryReader index2 = test.openReader(); - - TestIndexReader.assertIndexEquals(index1, index2); - - // verify that reopen() does not return a new reader instance - // in case the index has no changes - ReaderCouple couple = refreshReader(index2, false); - assertTrue(couple.refreshedReader == index2); - - couple = refreshReader(index2, test, 0, true); - index1.close(); - index1 = couple.newReader; - - DirectoryReader index2_refreshed = couple.refreshedReader; - index2.close(); - - // test if refreshed reader and newly opened reader return equal results - TestIndexReader.assertIndexEquals(index1, index2_refreshed); - - index2_refreshed.close(); - assertReaderClosed(index2, true, true); - assertReaderClosed(index2_refreshed, true, true); - - index2 = test.openReader(); - - for (int i = 1; i < 4; i++) { - - index1.close(); - couple = refreshReader(index2, test, i, true); - // refresh DirectoryReader - index2.close(); - - index2 = couple.refreshedReader; - index1 = couple.newReader; - TestIndexReader.assertIndexEquals(index1, index2); - } - - index1.close(); - index2.close(); - assertReaderClosed(index1, true, true); - assertReaderClosed(index2, true, true); - } - - private void performTestsWithExceptionInReopen(TestReopen test) throws Exception { - DirectoryReader index1 = test.openReader(); - DirectoryReader index2 = test.openReader(); - - TestIndexReader.assertIndexEquals(index1, index2); - - try { - refreshReader(index1, test, 0, true); - fail("Expected exception not thrown."); - } catch (Exception e) { - // expected exception - } - - // index2 should still be usable and unaffected by the failed reopen() call - TestIndexReader.assertIndexEquals(index1, index2); - - index1.close(); - index2.close(); - } - - public void testThreadSafety() throws Exception { - final Directory dir = newDirectory(); - // NOTE: this also controls the number of threads! - final int n = _TestUtil.nextInt(random, 20, 40); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - for (int i = 0; i < n; i++) { - writer.addDocument(createDocument(i, 3)); - } - writer.forceMerge(1); - writer.close(); - - final TestReopen test = new TestReopen() { - @Override - protected void modifyIndex(int i) throws IOException { - IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - modifier.addDocument(createDocument(n + i, 6)); - modifier.close(); - } - - @Override - protected DirectoryReader openReader() throws IOException { - return DirectoryReader.open(dir); - } - }; - - final List readers = Collections.synchronizedList(new ArrayList()); - DirectoryReader firstReader = DirectoryReader.open(dir); - DirectoryReader reader = firstReader; - final Random rnd = random; - - ReaderThread[] threads = new ReaderThread[n]; - final Set readersToClose = Collections.synchronizedSet(new HashSet()); - - for (int i = 0; i < n; i++) { - if (i % 2 == 0) { - DirectoryReader refreshed = DirectoryReader.openIfChanged(reader); - if (refreshed != null) { - readersToClose.add(reader); - reader = refreshed; - } - } - final DirectoryReader r = reader; - - final int index = i; - - ReaderThreadTask task; - - if (i < 4 || (i >=10 && i < 14) || i > 18) { - task = new ReaderThreadTask() { - - @Override - public void run() throws Exception { - while (!stopped) { - if (index % 2 == 0) { - // refresh reader synchronized - ReaderCouple c = (refreshReader(r, test, index, true)); - readersToClose.add(c.newReader); - readersToClose.add(c.refreshedReader); - readers.add(c); - // prevent too many readers - break; - } else { - // not synchronized - DirectoryReader refreshed = DirectoryReader.openIfChanged(r); - if (refreshed == null) { - refreshed = r; - } - - IndexSearcher searcher = newSearcher(refreshed); - ScoreDoc[] hits = searcher.search( - new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))), - null, 1000).scoreDocs; - if (hits.length > 0) { - searcher.doc(hits[0].doc); - } - if (refreshed != r) { - refreshed.close(); - } - } - synchronized(this) { - wait(_TestUtil.nextInt(random, 1, 100)); - } - } - } - - }; - } else { - task = new ReaderThreadTask() { - @Override - public void run() throws Exception { - while (!stopped) { - int numReaders = readers.size(); - if (numReaders > 0) { - ReaderCouple c = readers.get(rnd.nextInt(numReaders)); - TestIndexReader.assertIndexEquals(c.newReader, c.refreshedReader); - } - - synchronized(this) { - wait(_TestUtil.nextInt(random, 1, 100)); - } - } - } - }; - } - - threads[i] = new ReaderThread(task); - threads[i].start(); - } - - synchronized(this) { - wait(1000); - } - - for (int i = 0; i < n; i++) { - if (threads[i] != null) { - threads[i].stopThread(); - } - } - - for (int i = 0; i < n; i++) { - if (threads[i] != null) { - threads[i].join(); - if (threads[i].error != null) { - String msg = "Error occurred in thread " + threads[i].getName() + ":\n" + threads[i].error.getMessage(); - fail(msg); - } - } - - } - - for (final DirectoryReader readerToClose : readersToClose) { - readerToClose.close(); - } - - firstReader.close(); - reader.close(); - - for (final DirectoryReader readerToClose : readersToClose) { - assertReaderClosed(readerToClose, true, true); - } - - assertReaderClosed(reader, true, true); - assertReaderClosed(firstReader, true, true); - - dir.close(); - } - - private static class ReaderCouple { - ReaderCouple(DirectoryReader r1, DirectoryReader r2) { - newReader = r1; - refreshedReader = r2; - } - - DirectoryReader newReader; - DirectoryReader refreshedReader; - } - - private abstract static class ReaderThreadTask { - protected volatile boolean stopped; - public void stop() { - this.stopped = true; - } - - public abstract void run() throws Exception; - } - - private static class ReaderThread extends Thread { - private ReaderThreadTask task; - private Throwable error; - - - ReaderThread(ReaderThreadTask task) { - this.task = task; - } - - public void stopThread() { - this.task.stop(); - } - - @Override - public void run() { - try { - this.task.run(); - } catch (Throwable r) { - r.printStackTrace(System.out); - this.error = r; - } - } - } - - private Object createReaderMutex = new Object(); - - private ReaderCouple refreshReader(DirectoryReader reader, boolean hasChanges) throws IOException { - return refreshReader(reader, null, -1, hasChanges); - } - - ReaderCouple refreshReader(DirectoryReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException { - synchronized (createReaderMutex) { - DirectoryReader r = null; - if (test != null) { - test.modifyIndex(modify); - r = test.openReader(); - } - - DirectoryReader refreshed = null; - try { - refreshed = DirectoryReader.openIfChanged(reader); - if (refreshed == null) { - refreshed = reader; - } - } finally { - if (refreshed == null && r != null) { - // Hit exception -- close opened reader - r.close(); - } - } - - if (hasChanges) { - if (refreshed == reader) { - fail("No new DirectoryReader instance created during refresh."); - } - } else { - if (refreshed != reader) { - fail("New DirectoryReader instance created during refresh even though index had no changes."); - } - } - - return new ReaderCouple(r, refreshed); - } - } - - public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException { - IndexWriter.unlock(dir); - IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random, - TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setMergePolicy(new LogDocMergePolicy())); - - for (int i = 0; i < 100; i++) { - w.addDocument(createDocument(i, 4)); - if (multiSegment && (i % 10) == 0) { - w.commit(); - } - } - - if (!multiSegment) { - w.forceMerge(1); - } - - w.close(); - - DirectoryReader r = DirectoryReader.open(dir); - if (multiSegment) { - assertTrue(r.getSequentialSubReaders().length > 1); - } else { - assertTrue(r.getSequentialSubReaders().length == 1); - } - r.close(); - } - - public static Document createDocument(int n, int numFields) { - StringBuilder sb = new StringBuilder(); - Document doc = new Document(); - sb.append("a"); - sb.append(n); - FieldType customType2 = new FieldType(TextField.TYPE_STORED); - customType2.setTokenized(false); - customType2.setOmitNorms(true); - FieldType customType3 = new FieldType(); - customType3.setStored(true); - doc.add(new Field("field1", sb.toString(), TextField.TYPE_STORED)); - doc.add(new Field("fielda", sb.toString(), customType2)); - doc.add(new Field("fieldb", sb.toString(), customType3)); - sb.append(" b"); - sb.append(n); - for (int i = 1; i < numFields; i++) { - doc.add(new Field("field" + (i+1), sb.toString(), TextField.TYPE_STORED)); - } - return doc; - } - - static void modifyIndex(int i, Directory dir) throws IOException { - switch (i) { - case 0: { - if (VERBOSE) { - System.out.println("TEST: modify index"); - } - IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - w.deleteDocuments(new Term("field2", "a11")); - w.deleteDocuments(new Term("field2", "b30")); - w.close(); - break; - } - case 1: { - IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - w.forceMerge(1); - w.close(); - break; - } - case 2: { - IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - w.addDocument(createDocument(101, 4)); - w.forceMerge(1); - w.addDocument(createDocument(102, 4)); - w.addDocument(createDocument(103, 4)); - w.close(); - break; - } - case 3: { - IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - w.addDocument(createDocument(101, 4)); - w.close(); - break; - } - } - } - - static void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) { - assertEquals(0, reader.getRefCount()); - - if (checkNormsClosed && reader instanceof AtomicReader) { - // TODO: should we really assert something here? we check for open files and this is obselete... - // assertTrue(((SegmentReader) reader).normsClosed()); - } - - if (checkSubReaders && reader instanceof CompositeReader) { - IndexReader[] subReaders = ((CompositeReader) reader).getSequentialSubReaders(); - for (int i = 0; i < subReaders.length; i++) { - assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); - } - } - } - - /* - private void assertReaderOpen(DirectoryReader reader) { - reader.ensureOpen(); - - if (reader instanceof DirectoryReader) { - DirectoryReader[] subReaders = reader.getSequentialSubReaders(); - for (int i = 0; i < subReaders.length; i++) { - assertReaderOpen(subReaders[i]); - } - } - } - */ - - private void assertRefCountEquals(int refCount, DirectoryReader reader) { - assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount()); - } - - - private abstract static class TestReopen { - protected abstract DirectoryReader openReader() throws IOException; - protected abstract void modifyIndex(int i) throws IOException; - } - - private static class KeepAllCommits implements IndexDeletionPolicy { - public void onInit(List commits) { - } - public void onCommit(List commits) { - } - } - - public void testReopenOnCommit() throws Throwable { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter( - dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setIndexDeletionPolicy(new KeepAllCommits()). - setMaxBufferedDocs(-1). - setMergePolicy(newLogMergePolicy(10)) - ); - for(int i=0;i<4;i++) { - Document doc = new Document(); - doc.add(newField("id", ""+i, StringField.TYPE_UNSTORED)); - writer.addDocument(doc); - Map data = new HashMap(); - data.put("index", i+""); - writer.commit(data); - } - for(int i=0;i<4;i++) { - writer.deleteDocuments(new Term("id", ""+i)); - Map data = new HashMap(); - data.put("index", (4+i)+""); - writer.commit(data); - } - writer.close(); - - DirectoryReader r = DirectoryReader.open(dir); - assertEquals(0, r.numDocs()); - - Collection commits = DirectoryReader.listCommits(dir); - for (final IndexCommit commit : commits) { - DirectoryReader r2 = DirectoryReader.openIfChanged(r, commit); - assertNotNull(r2); - assertTrue(r2 != r); - - final Map s = commit.getUserData(); - final int v; - if (s.size() == 0) { - // First commit created by IW - v = -1; - } else { - v = Integer.parseInt(s.get("index")); - } - if (v < 4) { - assertEquals(1+v, r2.numDocs()); - } else { - assertEquals(7-v, r2.numDocs()); - } - r.close(); - r = r2; - } - r.close(); - dir.close(); - } -} Index: lucene/core/src/test/org/apache/lucene/index/TestMultiReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestMultiReader.java (revision 1243051) +++ lucene/core/src/test/org/apache/lucene/index/TestMultiReader.java (working copy) @@ -1,56 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper; - -public class TestMultiReader extends TestDirectoryReader { - - // TODO: files are never fsynced if you do what this test is doing, - // so the checkindex is disabled. - @Override - protected Directory createDirectory() throws IOException { - MockDirectoryWrapper mdw = newDirectory(); - mdw.setCheckIndexOnClose(false); - return mdw; - } - - @Override - protected IndexReader openReader() throws IOException { - IndexReader reader; - - sis.read(dir); - SegmentReader reader1 = new SegmentReader(sis.info(0), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); - SegmentReader reader2 = new SegmentReader(sis.info(1), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); - readers[0] = reader1; - readers[1] = reader2; - assertTrue(reader1 != null); - assertTrue(reader2 != null); - - reader = new MultiReader(readers); - - assertTrue(dir != null); - assertTrue(sis != null); - - return reader; - } - -}