Index: src/test/org/apache/lucene/index/TestFieldCacheReopen.java =================================================================== --- src/test/org/apache/lucene/index/TestFieldCacheReopen.java (revision 0) +++ src/test/org/apache/lucene/index/TestFieldCacheReopen.java (revision 0) @@ -0,0 +1,197 @@ +package org.apache.lucene.index; + +import java.io.File; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.SimpleFSDirectory; + +/** + * Test getCachedFieldValue in IndexReader and subclasses. Also tests to make + * sure the per-segment field caches do not get recreated on a reopen. + * + * This tests IndexReader, DirectoryReader and SegmentReader MultiReader and + * ParallelReader both currently use IndexReader.getCachedFieldValue(). It would + * make some sense for MultiReader to delegate to getCachedFieldValue to its + * subreaders. + */ +public class TestFieldCacheReopen extends TestCase { + static final String fieldName = "field88"; + + static final String data[] = { "Hello, world!", + "All work and no play makes jack a dull boy", "Hello, tellus!", + "All work and no play makes danny a dull boy", "Hello, earth!", + "All work and no play makes wendy a dull girl" }; + + static final String dat2[] = { "Hello, mars!", + "All your base are belong to us." }; + + public void test() throws Exception { + boolean useRam = true; + Directory dir; + if (useRam) + dir = new RAMDirectory(); + else + dir = new SimpleFSDirectory(new File("/tmp/foo/")); + + IndexWriter iw = getWriter(dir, true); + addDocument(iw, data[0]); + addDocument(iw, data[1]); + iw.close(); + + iw = getWriter(dir, false); + addDocument(iw, data[2]); + addDocument(iw, data[3]); + iw.close(); + + iw = getWriter(dir, false); + addDocument(iw, data[4]); + addDocument(iw, data[5]); + iw.close(); + + // We should have 6 docs in the index now. + boolean readOnly = false; + IndexReader reader = IndexReader.open(dir, readOnly); + int maxDoc = reader.maxDoc(); + assertEquals(6, maxDoc); + + final FieldCache cache = FieldCache.DEFAULT; + + checkInitialData(cache, reader); + + // Get the individual segment caches as of this point. + List firstReaderCache = getCaches(reader, cache); + assertEquals(3, firstReaderCache.size()); + + // Get the overall cache for the field. This is probably NOT + // how we actually want to get the cache. + String[] firstCache = cache.getStrings(reader, fieldName); + assertEquals(6, firstCache.length); + + IndexReader newReader; + + // Nothing has been added, should get the same IndexReader back from reopen. + newReader = reader.reopen(); + assertEquals(reader, newReader); + + iw = new IndexWriter(dir, new WhitespaceAnalyzer(), false, + IndexWriter.MaxFieldLength.UNLIMITED); + addDocument(iw, dat2[0]); + addDocument(iw, dat2[1]); + iw.close(); + + // There have been changed, we should have a new index reader. + newReader = reader.reopen(); + assertTrue(reader != newReader); + + String[] sameCache = cache.getStrings(reader, fieldName); + assertEquals(firstCache, sameCache); + + // This is the wrong way to get field caches that survive a reopen. + // This gets a field cache indexed by the top level IndexReader, + // which changes with every reopen. + String[] diffCache = cache.getStrings(newReader, fieldName); + + assertTrue("Got old cache, after add and reopen", firstCache != diffCache); + assertEquals(8, newReader.maxDoc()); + assertEquals(8, diffCache.length); + + // Get the individual subreader caches + IndexReader[] readers = newReader.getSequentialSubReaders(); + int sumOfCaches = 0; + for (int i = 0; i < readers.length; i++) { + String[] subCache = cache.getStrings(readers[i], fieldName); + sumOfCaches += subCache.length; + } + assertEquals(8, sumOfCaches); + + // Get the individual segment caches as of this point. + List secondReaderCache = getCaches(newReader, cache); + for (int i = 0; i < firstReaderCache.size(); i++) { + // We want ==, not equals + assertTrue(firstReaderCache.get(i) == secondReaderCache.get(i)); + } + + // Check to see that the content of the caches match. + // This will test our getCachedFieldValue method in + // IndexReader and DirectoryReader + for (int docNumber = 0; docNumber < 6; docNumber++) { + // We want ==, not equals + String c = newReader.getCachedFieldValue(cache, fieldName, docNumber); + assertTrue(reader.getCachedFieldValue(cache, fieldName, docNumber) == newReader + .getCachedFieldValue(cache, fieldName, docNumber)); + } + + iw = getWriter(dir, false); + iw.optimize(); + newReader = newReader.reopen(); + checkInitialData(cache, newReader); + checkSecondaryData(cache, newReader); + } + + private IndexWriter getWriter(Directory dir, boolean create) + throws CorruptIndexException, LockObtainFailedException, IOException { + IndexWriter iw; + iw = new IndexWriter(dir, new WhitespaceAnalyzer(), create, + IndexWriter.MaxFieldLength.UNLIMITED); + iw.setUseCompoundFile(false); + return iw; + } + + private void addDocument(IndexWriter iw, String text) throws IOException { + Document doc = new Document(); + doc.add(new Field(fieldName, text, Field.Store.YES, + Field.Index.NOT_ANALYZED)); + iw.addDocument(doc); + } + + /** + * Create a list of field caches (one per segment) from the given reader. + * + * @param reader + * @param cache + * @return A List containing one entry for each segment + * @throws IOException + */ + List getCaches(IndexReader reader, FieldCache cache) throws IOException { + List l = new LinkedList(); + + IndexReader[] readers = reader.getSequentialSubReaders(); + for (int i = 0; i < readers.length; i++) { + String[] subCache = cache.getStrings(readers[i], fieldName); + l.add(subCache); + } + return l; + } + + void checkInitialData(FieldCache cache, IndexReader reader) + throws CorruptIndexException, IOException { + for (int docNumber = 0; docNumber < data.length; docNumber++) { + assertEquals(data[docNumber], reader.getCachedFieldValue(cache, + fieldName, docNumber)); + } + + } + + void checkSecondaryData(FieldCache cache, IndexReader reader) + throws CorruptIndexException, IOException { + for (int docNumber = data.length; docNumber < data.length + dat2.length; docNumber++) { + assertEquals(dat2[docNumber - data.length], reader.getCachedFieldValue( + cache, fieldName, docNumber)); + } + + } +} Index: src/test/org/apache/lucene/index/TestFieldCacheReopenStress.java =================================================================== --- src/test/org/apache/lucene/index/TestFieldCacheReopenStress.java (revision 0) +++ src/test/org/apache/lucene/index/TestFieldCacheReopenStress.java (revision 0) @@ -0,0 +1,138 @@ +package org.apache.lucene.index; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FilterIndexReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.SimpleFSDirectory; + +/** + * Test getFieldValueCache. This version is a bit long for a unit test. We are + * just going to generate random docs with random commits and then see if the + * cache matches the stored value. + **/ +public class TestFieldCacheReopenStress extends TestCase { + static final String fieldName = "field88"; + + static final String data[] = "word another sky blue pond ocean screen orange banana apple pear tree leaf pineapple kiwi Hello, world All work and no play makes jack a dull boy tellus Hello earth wendy girl" + .split(" "); + + Random r = new Random(); + + public void test() throws Exception { + boolean useRam = true; + Directory dir; + // Repeat the whole process a few times. + for (int rep = 0; rep < 3; rep++) { + if (useRam) + dir = new RAMDirectory(); + else + dir = new SimpleFSDirectory(new File("/tmp/foo/")); + + // Create the directory + IndexWriter iw = getWriter(dir, true); + iw.close(); + // Make sure some merging will go on, but not every segment will be + // merged. + iw.setMergeFactor(29); + generateRandomDocs(dir); + + final FieldCache cache = FieldCache.DEFAULT; + + compareStoredToCache(dir, cache); + iw = getWriter(dir, false); + iw.optimize(); + iw.close(); + compareStoredToCache(dir, cache); + for (int more = 0; more < 7; more++) { + // More docs + generateRandomDocs(dir); + // Does the cache still match? + compareStoredToCache(dir, cache); + } + } + } + + private void generateRandomDocs(Directory dir) throws CorruptIndexException, + LockObtainFailedException, IOException { + IndexWriter iw; + // generate an index with a random number of segments with a random number + // of random docs. + int nSegments = r.nextInt(61); + for (int seg = 0; seg < nSegments; seg++) { + iw = getWriter(dir, false); + int docs = r.nextInt(57); + for (int d = 0; d < docs; d++) { + String doc = makeDoc(); + addDocument(iw, doc); + } + iw.close(); + } + } + + /** See if the cache has remained consistent, by checking it against the + * stored value in the field. + * @param dir + * @param cache + * @throws CorruptIndexException + * @throws IOException + */ + private void compareStoredToCache(Directory dir, final FieldCache cache) + throws CorruptIndexException, IOException { + IndexReader reader = IndexReader.open(dir, true); + // Use a FilterReader, just so we pass through its + // implementation of getCachedFieldValue, too. + FilterIndexReader freader = new FilterIndexReader(reader); + int maxDoc = reader.maxDoc(); + for (int docNumber = 0; docNumber < maxDoc; docNumber++) { + Document d = reader.document(docNumber); + assertEquals(d.get(fieldName), freader.getCachedFieldValue(cache, + fieldName, docNumber)); + } + reader.close(); + } + + private IndexWriter getWriter(Directory dir, boolean create) + throws CorruptIndexException, LockObtainFailedException, IOException { + IndexWriter iw; + iw = new IndexWriter(dir, new WhitespaceAnalyzer(), create, + IndexWriter.MaxFieldLength.UNLIMITED); + iw.setUseCompoundFile(false); + return iw; + } + + private void addDocument(IndexWriter iw, String text) throws IOException { + Document doc = new Document(); + doc.add(new Field(fieldName, text, Field.Store.YES, + Field.Index.NOT_ANALYZED)); + iw.addDocument(doc); + } + + /** Creates a document, consisting of random text. + * + * @return The randomly generated document. + */ + String makeDoc() { + int count = 31; + StringBuilder sb = new StringBuilder(count); + for (int i = 0; i < count; i++) { + sb.append(data[r.nextInt(data.length)]); + sb.append(" "); + } + return sb.toString(); + } + +} Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 822965) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -1358,4 +1359,10 @@ public int getTermInfosIndexDivisor() { return core.termsIndexDivisor; } + // @Override + public String getCachedFieldValue(FieldCache cache, String fieldName, int n) throws CorruptIndexException, IOException { + ensureOpen(); + String [] fieldCache = cache.getStrings(this, fieldName); + return fieldCache[n]; + } } Index: src/java/org/apache/lucene/index/MultiReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiReader.java (revision 822965) +++ src/java/org/apache/lucene/index/MultiReader.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.index.DirectoryReader.MultiTermEnum; import org.apache.lucene.index.DirectoryReader.MultiTermPositions; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.FieldCache; /** An IndexReader which reads multiple indexes, appending their content. * @@ -404,4 +405,14 @@ public IndexReader[] getSequentialSubReaders() { return subReaders; } + /** + * Looks up a cached value for a field. This gets the value from the subReader + */ + // @Override + public String getCachedFieldValue(FieldCache cache, String fieldName, int n) + throws CorruptIndexException, IOException { + ensureOpen(); + int i = readerIndex(n); // find segment num + return subReaders[i].getCachedFieldValue(cache, fieldName, n - starts[i]); + } } Index: src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- src/java/org/apache/lucene/index/DirectoryReader.java (revision 822965) +++ src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -32,6 +32,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; @@ -1215,4 +1216,15 @@ return ((TermPositions) current).isPayloadAvailable(); } } + /** + * Looks up a cached value for a field. This gets the value from the subReader, so it should not + * have to be regenerated when documents are added to the index. + */ + // @Override + public String getCachedFieldValue(FieldCache cache, String fieldName, int n) + throws CorruptIndexException, IOException { + ensureOpen(); + int i = readerIndex(n); // find segment num + return subReaders[i].getCachedFieldValue(cache, fieldName, n - starts[i]); + } } Index: src/java/org/apache/lucene/index/FilterIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/FilterIndexReader.java (revision 822965) +++ src/java/org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import java.io.IOException; @@ -242,4 +243,13 @@ public IndexReader[] getSequentialSubReaders() { return in.getSequentialSubReaders(); } + /** + * Looks up a cached value for a field. This just gets the value from the contained reader. + */ + // @Override + public String getCachedFieldValue(FieldCache cache, String fieldName, int n) + throws CorruptIndexException, IOException { + ensureOpen(); + return in.getCachedFieldValue(cache, fieldName, n); + } } Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 822965) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.*; @@ -1503,4 +1504,12 @@ public void setDisableFakeNorms(boolean disableFakeNorms) { this.disableFakeNorms = disableFakeNorms; } + /** Expert: Gets a value from the field cache + * @param cache The field cache to fetch from + * @param fieldName The name of the field + * @param docNumber The document number + */ + public String getCachedFieldValue(FieldCache cache, String fieldName, int docNumber)throws CorruptIndexException, IOException { + return cache.getStrings(this, fieldName)[docNumber]; + } }