Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java (revision 1036589) +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -1334,12 +1334,14 @@ assertEquals("Different field names.", it1.next(), it2.next()); } + IndexReader index1Slow = SlowMultiReaderWrapper.wrap(index1); + IndexReader index2Slow = SlowMultiReaderWrapper.wrap(index2); // check norms it1 = fields1.iterator(); while (it1.hasNext()) { String curField = it1.next(); - byte[] norms1 = index1.norms(curField); - byte[] norms2 = index2.norms(curField); + byte[] norms1 = index1Slow.norms(curField); + byte[] norms2 = index2Slow.norms(curField); if (norms1 != null && norms2 != null) { assertEquals(norms1.length, norms2.length); Index: lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java (revision 1036589) +++ lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java (working copy) @@ -149,7 +149,7 @@ //IndexReader reader = new TestReader(IndexReader.open(directory, true)); Directory target = newDirectory(); writer = new IndexWriter(target, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); - IndexReader reader = new TestReader(IndexReader.open(directory, true)); + IndexReader reader = new TestReader(SlowMultiReaderWrapper.wrap(IndexReader.open(directory, true))); writer.addIndexes(reader); writer.close(); reader.close(); Index: lucene/src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (revision 1036589) +++ lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (working copy) @@ -173,6 +173,8 @@ } public static void checkNorms(IndexReader reader) throws IOException { + // XXX + reader = SlowMultiReaderWrapper.wrap(reader); // test omit norms for (int i=0; i subReaderToSlice = new HashMap(); private boolean[] decrefOnClose; // remember which subreaders to decRef on close - private Map normsCache = new HashMap(); private int maxDoc = 0; private int numDocs = -1; private boolean hasDeletions = false; @@ -316,45 +315,18 @@ @Override public synchronized byte[] norms(String field) throws IOException { - ensureOpen(); - byte[] bytes = normsCache.get(field); - if (bytes != null) - return bytes; // cache hit - if (!hasNorms(field)) - return null; - - bytes = new byte[maxDoc()]; - for (int i = 0; i < subReaders.length; i++) - subReaders[i].norms(field, bytes, starts[i]); - normsCache.put(field, bytes); // update cache - return bytes; + throw new UnsupportedOperationException("Use SlowMultiReaderWrapper instead."); } @Override public synchronized void norms(String field, byte[] result, int offset) throws IOException { - ensureOpen(); - byte[] bytes = normsCache.get(field); - for (int i = 0; i < subReaders.length; i++) // read from segments - subReaders[i].norms(field, result, offset + starts[i]); - - if (bytes==null && !hasNorms(field)) { - Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f)); - } else if (bytes != null) { // cache hit - System.arraycopy(bytes, 0, result, offset, maxDoc()); - } else { - for (int i = 0; i < subReaders.length; i++) { // read from segments - subReaders[i].norms(field, result, offset + starts[i]); - } - } + throw new UnsupportedOperationException("Use SlowMultiReaderWrapper instead."); } @Override protected void doSetNorm(int n, String field, byte value) throws CorruptIndexException, IOException { - synchronized (normsCache) { - normsCache.remove(field); // clear cache - } int i = readerIndex(n); // find segment num subReaders[i].setNorm(n-starts[i], field, value); // dispatch } Index: lucene/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DirectoryReader.java (revision 1036589) +++ lucene/src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -65,7 +65,6 @@ private SegmentReader[] subReaders; private int[] starts; // 1st docno for each segment private final Map subReaderToSlice = new HashMap(); - private Map normsCache = new HashMap(); private int maxDoc = 0; private int numDocs = -1; private boolean hasDeletions = false; @@ -189,7 +188,7 @@ /** This constructor is only used for {@link #reopen()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, - Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs) throws IOException { + boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = infos; @@ -277,38 +276,6 @@ // initialize the readers to calculate maxDoc before we try to reuse the old normsCache initialize(newReaders); - - // try to copy unchanged norms from the old normsCache to the new one - if (oldNormsCache != null) { - for (Map.Entry entry: oldNormsCache.entrySet()) { - String field = entry.getKey(); - if (!hasNorms(field)) { - continue; - } - - byte[] oldBytes = entry.getValue(); - - byte[] bytes = new byte[maxDoc()]; - - for (int i = 0; i < subReaders.length; i++) { - Integer oldReaderIndex = segmentReaders.get(subReaders[i].getSegmentName()); - - // this SegmentReader was not re-opened, we can copy all of its norms - if (oldReaderIndex != null && - (oldReaders[oldReaderIndex.intValue()] == subReaders[i] - || oldReaders[oldReaderIndex.intValue()].norms.get(field) == subReaders[i].norms.get(field))) { - // we don't have to synchronize here: either this constructor is called from a SegmentReader, - // in which case no old norms cache is present, or it is called from MultiReader.reopen(), - // which is synchronized - System.arraycopy(oldBytes, oldStarts[oldReaderIndex.intValue()], bytes, starts[i], starts[i+1] - starts[i]); - } else { - subReaders[i].norms(field, bytes, starts[i]); - } - } - - normsCache.put(field, bytes); // update cache - } - } } /** {@inheritDoc} */ @@ -500,7 +467,7 @@ private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { DirectoryReader reader; - reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, openReadOnly, doClone, termInfosIndexDivisor, codecs); + reader = new DirectoryReader(directory, infos, subReaders, starts, openReadOnly, doClone, termInfosIndexDivisor, codecs); return reader; } @@ -640,41 +607,18 @@ @Override public synchronized byte[] norms(String field) throws IOException { ensureOpen(); - byte[] bytes = normsCache.get(field); - if (bytes != null) - return bytes; // cache hit - if (!hasNorms(field)) - return null; - - bytes = new byte[maxDoc()]; - for (int i = 0; i < subReaders.length; i++) - subReaders[i].norms(field, bytes, starts[i]); - normsCache.put(field, bytes); // update cache - return bytes; + throw new UnsupportedOperationException("Use SlowMultiReaderWrapper instead"); } @Override public synchronized void norms(String field, byte[] result, int offset) throws IOException { - ensureOpen(); - byte[] bytes = normsCache.get(field); - if (bytes==null && !hasNorms(field)) { - Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f)); - } else if (bytes != null) { // cache hit - System.arraycopy(bytes, 0, result, offset, maxDoc()); - } else { - for (int i = 0; i < subReaders.length; i++) { // read from segments - subReaders[i].norms(field, result, offset + starts[i]); - } - } + throw new UnsupportedOperationException("Use SlowMultiReaderWrapper instead"); } @Override protected void doSetNorm(int n, String field, byte value) throws CorruptIndexException, IOException { - synchronized (normsCache) { - normsCache.remove(field); // clear cache - } int i = readerIndex(n); // find segment num subReaders[i].setNorm(n-starts[i], field, value); // dispatch } @@ -867,7 +811,6 @@ @Override protected synchronized void doClose() throws IOException { IOException ioe = null; - normsCache = null; for (int i = 0; i < subReaders.length; i++) { // try to close each reader, even if an exception is thrown try { Index: lucene/src/java/org/apache/lucene/index/FilterIndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (revision 1036589) +++ lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -279,7 +279,7 @@ @Override public Bits getDeletedDocs() { - return MultiFields.getDeletedDocs(in); + return in.getDeletedDocs(); } @Override @@ -415,12 +415,12 @@ @Override public IndexReader[] getSequentialSubReaders() { - return null; + return in.getSequentialSubReaders(); } @Override public Fields fields() throws IOException { - return MultiFields.getFields(in); + return in.fields(); } /** If the subclass of FilteredIndexReader modifies the Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision 1036589) +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -692,7 +692,15 @@ // the buffer is too small for the current segment normBuffer = new byte[maxDoc]; } - reader.norms(fi.name, normBuffer, 0); + // collect the norms from the sub + final List subs = new ArrayList(); + ReaderUtil.gatherSubReaders(subs, reader); + int end = 0; + for (IndexReader sub : subs) { + int start = end; + sub.norms(fi.name, normBuffer, start); + end += sub.maxDoc(); + } if (!reader.hasDeletions()) { //optimized case for segments without deleted docs output.writeBytes(normBuffer, maxDoc); Index: lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (revision 1036589) +++ lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (working copy) @@ -18,8 +18,13 @@ */ import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.ArrayList; +import java.util.Map; + +import org.apache.lucene.search.Similarity; import org.apache.lucene.util.Bits; import org.apache.lucene.util.ReaderUtil; @@ -60,12 +65,23 @@ } else if (subs.size() == 1) { return subs.get(0); } else { - return new SlowMultiReaderWrapper(reader); + return new SlowMultiReaderWrapper(reader, subs.toArray(new IndexReader[subs.size()])); } } - private SlowMultiReaderWrapper(IndexReader other) throws IOException { + private final IndexReader subReaders[]; + private final int starts[]; + private Map normsCache = new HashMap(); + + private SlowMultiReaderWrapper(IndexReader other, IndexReader subReaders[]) throws IOException { super(other); + this.subReaders = subReaders; + this.starts = new int[subReaders.length + 1]; + int maxDoc = 0; + for (int i = 0; i < subReaders.length; i++) { + starts[i] = maxDoc; + maxDoc += subReaders[i].maxDoc(); + } } @Override @@ -80,6 +96,56 @@ @Override public void doClose() throws IOException { + // nocommit: don't we want to null out the norms cache here? throw new UnsupportedOperationException("please call close on the original reader instead"); } + + @Override + public IndexReader[] getSequentialSubReaders() { + return null; + } + + @Override + public synchronized byte[] norms(String field) throws IOException { + ensureOpen(); + byte[] bytes = normsCache.get(field); + if (bytes != null) + return bytes; // cache hit + if (!hasNorms(field)) + return null; + + bytes = new byte[maxDoc()]; + for (int i = 0; i < subReaders.length; i++) + subReaders[i].norms(field, bytes, starts[i]); + normsCache.put(field, bytes); // update cache + return bytes; + } + + @Override + public synchronized void norms(String field, byte[] result, int offset) + throws IOException { + ensureOpen(); + byte[] bytes = normsCache.get(field); + for (int i = 0; i < subReaders.length; i++) // read from segments + subReaders[i].norms(field, result, offset + starts[i]); + + if (bytes==null && !hasNorms(field)) { + Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f)); + } else if (bytes != null) { // cache hit + System.arraycopy(bytes, 0, result, offset, maxDoc()); + } else { + for (int i = 0; i < subReaders.length; i++) { // read from segments + subReaders[i].norms(field, result, offset + starts[i]); + } + } + } + + @Override + protected void doSetNorm(int n, String field, byte value) + throws CorruptIndexException, IOException { + synchronized(normsCache) { + normsCache.remove(field); + } + in.doSetNorm(n, field, value); + } } Index: lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (revision 1036589) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (working copy) @@ -179,8 +179,8 @@ OpenBitSet dels; OpenBitSet oldDels = null; - public FakeDeleteIndexReader(IndexReader in) { - super(in); + public FakeDeleteIndexReader(IndexReader in) throws IOException { + super(SlowMultiReaderWrapper.wrap(in)); dels = new OpenBitSet(in.maxDoc()); if (in.hasDeletions()) { oldDels = new OpenBitSet(in.maxDoc());