Index: src/test/org/apache/lucene/index/TestIndexReaderClone.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReaderClone.java (revision 0) +++ src/test/org/apache/lucene/index/TestIndexReaderClone.java (revision 0) @@ -0,0 +1,48 @@ +package org.apache.lucene.index; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; + +public class TestIndexReaderClone extends LuceneTestCase { + public void testDeletedDocsReferenceCounting() throws Exception { + final Directory dir1 = new RAMDirectory(); + TestIndexReaderReopen.createIndex(dir1, false); + SegmentReader segmentReader = (SegmentReader)IndexReader.open(dir1); + assertDelDocsRefCountEquals(0, segmentReader); + + // we deleted a document, so there is now a deletedDocs bitvector and a reference to it + segmentReader.deleteDocument(1); + assertDelDocsRefCountEquals(1, segmentReader); + + // the cloned segmentreader should have 2 references, 1 to itself, and 1 to the original segmentreader + SegmentReader clonedSegmentReader = (SegmentReader)segmentReader.clone(); + assertDelDocsRefCountEquals(2, segmentReader); + // deleting a document creates a new deletedDocs bitvector, the refs goes to 1 + clonedSegmentReader.deleteDocument(2); + assertDelDocsRefCountEquals(1, segmentReader); + assertDelDocsRefCountEquals(1, clonedSegmentReader); + + // deleting a doc from the original segmentreader should throw an exception + try { + segmentReader.deleteDocument(4); + } catch (LockObtainFailedException lbfe) {} + segmentReader.close(); + // try closing the original segment reader to see if it affects the clonedSegmentReader + clonedSegmentReader.deleteDocument(3); + clonedSegmentReader.flush(); + assertDelDocsRefCountEquals(1, clonedSegmentReader); + SegmentReader reopenedSegmentReader = (SegmentReader)clonedSegmentReader.reopen(); + SegmentReader cloneSegmentReader2 = (SegmentReader)reopenedSegmentReader.clone(); + + } + + public void testDeletedDocsClonedDocsDeletes() throws Exception { + + } + + private void assertDelDocsRefCountEquals(int refCount, SegmentReader reader) { + assertEquals(refCount, reader.deletedDocsCopyOnWriteRef.refCount()); + } +} Property changes on: src/test/org/apache/lucene/index/TestIndexReaderClone.java ___________________________________________________________________ Name: svn:mime-type + text/plain Name: svn:keywords + "Date Rev Author URL Id" Name: svn:eol-style + native Index: src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 724958) +++ src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -852,7 +852,7 @@ } } - private static void createIndex(Directory dir, boolean multiSegment) throws IOException { + public static void createIndex(Directory dir, boolean multiSegment) throws IOException { IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); w.setMergePolicy(new LogDocMergePolicy()); Index: src/java/org/apache/lucene/index/DirectoryIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/DirectoryIndexReader.java (revision 724958) +++ src/java/org/apache/lucene/index/DirectoryIndexReader.java (working copy) @@ -36,13 +36,13 @@ * "own" the directory, which means that they try to acquire a write lock * whenever index modifications are performed. */ -abstract class DirectoryIndexReader extends IndexReader { +abstract class DirectoryIndexReader extends IndexReader implements Cloneable { protected Directory directory; protected boolean closeDirectory; private IndexDeletionPolicy deletionPolicy; private SegmentInfos segmentInfos; - private Lock writeLock; + protected Lock writeLock; private boolean stale; private final HashSet synced = new HashSet(); @@ -138,11 +138,23 @@ return reader; } - + public final synchronized IndexReader reopen() throws CorruptIndexException, IOException { + return doReopen(false); + } + + public final synchronized Object clone() { + try { + return doReopen(true); + } catch (Exception ex) { + throw new RuntimeException("", ex); + } + } + + protected final synchronized IndexReader doReopen(final boolean doClone) throws CorruptIndexException, IOException { ensureOpen(); - if (this.hasChanges || this.isCurrent()) { + if (!doClone && (this.hasChanges || this.isCurrent())) { // this has changes, therefore we have the lock and don't need to reopen // OR: the index in the directory hasn't changed - nothing to do here return this; @@ -154,7 +166,7 @@ SegmentInfos infos = new SegmentInfos(); infos.read(directory, segmentFileName); - DirectoryIndexReader newReader = doReopen(infos); + DirectoryIndexReader newReader = doReopen(infos, doClone); if (DirectoryIndexReader.this != newReader) { newReader.init(directory, infos, closeDirectory, readOnly); @@ -197,7 +209,7 @@ /** * Re-opens the index using the passed-in SegmentInfos */ - protected abstract DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException; + protected abstract DirectoryIndexReader doReopen(SegmentInfos infos, boolean doReopen) throws CorruptIndexException, IOException; public void setDeletionPolicy(IndexDeletionPolicy deletionPolicy) { this.deletionPolicy = deletionPolicy; Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 724958) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -54,6 +54,8 @@ CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); BitVector deletedDocs = null; + Ref deletedDocsCopyOnWriteRef = new Ref(); + private Ref writeLockRef = new Ref(); private boolean deletedDocsDirty = false; private boolean normsDirty = false; private boolean undeleteAll = false; @@ -64,7 +66,8 @@ private boolean rollbackUndeleteAll = false; private int rollbackPendingDeleteCount; private boolean readOnly; - + private boolean isCloned = false; + protected boolean releaseLockOnClone = true; IndexInput freqStream; IndexInput proxStream; @@ -79,6 +82,27 @@ // in case this is a re-opened reader private SegmentReader referencedSegmentReader = null; + static class Ref { + volatile private int refCount; + + public synchronized int refCount() { + return refCount; + } + + public synchronized void incRef() { + //assert refCount > 0; + refCount++; + } + + public synchronized void decRef() { + assert refCount > 0; + //if (refCount == 1) { + // close(); + // } + refCount--; + } + } + private class Norm { volatile int refCount; boolean useSingleNormStream; @@ -400,6 +424,8 @@ // NOTE: the bitvector is stored using the regular directory, not cfs if (hasDeletions(si)) { deletedDocs = new BitVector(directory(), si.getDelFileName()); + deletedDocsCopyOnWriteRef = new Ref(); + deletedDocsCopyOnWriteRef.incRef(); assert si.getDelCount() == deletedDocs.count() : "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count(); @@ -413,13 +439,23 @@ assert si.getDelCount() == 0; } - protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException { + //public Object clone() { + // BitVector deletedDocsBV = (BitVector)deletedDocs.clone(); + // doReopen(infos, true); + //doReopen(SegmentInfos infos, boolean doClone); + //} + + protected BitVector cloneBitVector(BitVector bv) { + return (BitVector)bv.clone(); + } + + protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos, boolean doClone) throws CorruptIndexException, IOException { DirectoryIndexReader newReader; if (infos.size() == 1) { SegmentInfo si = infos.info(0); if (segment.equals(si.name) && si.getUseCompoundFile() == SegmentReader.this.si.getUseCompoundFile()) { - newReader = reopenSegment(si); + newReader = reopenSegment(si, doClone); } else { // segment not referenced anymore, reopen not possible // or segment format changed @@ -427,20 +463,27 @@ } } else { if (readOnly) - return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null); + return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null, doClone); else - return new MultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null, false); + return new MultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null, false, doClone); } return newReader; } - synchronized SegmentReader reopenSegment(SegmentInfo si) throws CorruptIndexException, IOException { + synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone) throws CorruptIndexException, IOException { boolean deletionsUpToDate = (this.si.hasDeletions() == si.hasDeletions()) && (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName())); boolean normsUpToDate = true; + if (doClone && releaseLockOnClone) { + if (writeLock != null) { + writeLock.release(); + writeLock = null; + } + } + boolean[] fieldNormsChanged = new boolean[fieldInfos.size()]; if (normsUpToDate) { for (int i = 0; i < fieldInfos.size(); i++) { @@ -451,11 +494,11 @@ } } - if (normsUpToDate && deletionsUpToDate) { + if (normsUpToDate && deletionsUpToDate && !doClone) { return this; } - + // clone reader SegmentReader clone; if (readOnly) @@ -504,15 +547,27 @@ si.getDocStoreOffset(), si.docCount); } - - if (!deletionsUpToDate) { - // load deleted docs - clone.deletedDocs = null; - clone.loadDeletedDocs(); + if (doClone) { + clone.deletedDocs = cloneBitVector(deletedDocs); + deletedDocsCopyOnWriteRef.incRef(); + isCloned = true; // mark this segmentreader as cloned + /** + clone.writeLock = writeLock; + if (writeLockRef != null) { + clone.writeLockRef = writeLockRef; + writeLockRef.incRef(); + } + **/ } else { - clone.deletedDocs = this.deletedDocs; + if (!deletionsUpToDate) { + // load deleted docs + clone.deletedDocs = null; + clone.loadDeletedDocs(); + } else { + clone.deletedDocs = deletedDocs; + } } - + clone.deletedDocsCopyOnWriteRef = deletedDocsCopyOnWriteRef; clone.norms = new HashMap(); if (!normsUpToDate) { // load norms @@ -615,7 +670,14 @@ FieldsReader getFieldsReader() { return fieldsReader; } - + + //protected void acquireWriteLock() throws IOException { + // System.out.println("acquireWriteLock ref count: "+this.writeLockRef.refCount+" isCloned: "+isCloned); + // if (isCloned) throw new UnsupportedOperationException("this reader has been cloned and no updates are allowed"); + // if (writeLockRef.refCount() == 0) super.acquireWriteLock(); + + //} + protected void doClose() throws IOException { boolean hasReferencedReader = (referencedSegmentReader != null); @@ -689,8 +751,20 @@ } protected void doDelete(int docNum) { - if (deletedDocs == null) + if (deletedDocs == null) { deletedDocs = new BitVector(maxDoc()); + deletedDocsCopyOnWriteRef = new Ref(); + deletedDocsCopyOnWriteRef.incRef(); + } + // there is more than 1 SegmentReader with a reference to this + // deletedDocs BitVector so decRef the current copyOnWriteRef, + // clone the BitVector, create a new copyOnWriteRef + if (deletedDocsCopyOnWriteRef.refCount() > 1) { + deletedDocsCopyOnWriteRef.decRef(); + deletedDocs = cloneBitVector(deletedDocs); + deletedDocsCopyOnWriteRef = new Ref(); + deletedDocsCopyOnWriteRef.incRef(); + } deletedDocsDirty = true; undeleteAll = false; if (!deletedDocs.getAndSet(docNum)) Index: src/java/org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiSegmentReader.java (revision 724958) +++ src/java/org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -71,7 +71,7 @@ } /** This contructor is only used for {@link #reopen()} */ - MultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache, boolean readOnly) throws IOException { + MultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache, boolean readOnly, boolean doReopen) throws IOException { super(directory, infos, closeDirectory, readOnly); // we put the old SegmentReaders in a map, that allows us @@ -109,7 +109,7 @@ // this is a new reader; in case we hit an exception we can close it safely newReader = SegmentReader.get(readOnly, infos.info(i)); } else { - newReader = (SegmentReader) newReaders[i].reopenSegment(infos.info(i)); + newReader = (SegmentReader) newReaders[i].reopenSegment(infos.info(i), doReopen); } if (newReader == newReaders[i]) { // this reader will be shared between the old and the new one, @@ -195,15 +195,15 @@ starts[subReaders.length] = maxDoc; } - protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException { + protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos, boolean doReopen) throws CorruptIndexException, IOException { if (infos.size() == 1) { // The index has only one segment now, so we can't refresh the MultiSegmentReader. // Return a new [ReadOnly]SegmentReader instead return SegmentReader.get(readOnly, infos, infos.info(0), false); } else if (readOnly) { - return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache); + return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, doReopen); } else { - return new MultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, false); + return new MultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, false, doReopen); } } Index: src/java/org/apache/lucene/index/ReadOnlyMultiSegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/ReadOnlyMultiSegmentReader.java (revision 724958) +++ src/java/org/apache/lucene/index/ReadOnlyMultiSegmentReader.java (working copy) @@ -27,8 +27,8 @@ super(directory, sis, closeDirectory, true); } - ReadOnlyMultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache) throws IOException { - super(directory, infos, closeDirectory, oldReaders, oldStarts, oldNormsCache, true); + ReadOnlyMultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache, boolean doReopen) throws IOException { + super(directory, infos, closeDirectory, oldReaders, oldStarts, oldNormsCache, true, doReopen); } protected void acquireWriteLock() { Index: src/java/org/apache/lucene/util/BitVector.java =================================================================== --- src/java/org/apache/lucene/util/BitVector.java (revision 724958) +++ src/java/org/apache/lucene/util/BitVector.java (working copy) @@ -35,7 +35,7 @@ @version $Id$ */ -public final class BitVector { +public final class BitVector implements Cloneable { private byte[] bits; private int size; @@ -46,7 +46,19 @@ size = n; bits = new byte[(size >> 3) + 1]; } - + + BitVector(byte[] bits, int size, int count) { + this.bits = bits; + this.size = size; + this.count = count; + } + + public Object clone() { + byte[] copyBits = new byte[bits.length]; + System.arraycopy(bits, 0, copyBits, 0, bits.length); + return new BitVector(bits, size, count); + } + /** Sets the value of bit to one. */ public final void set(int bit) { if (bit >= size) {