Index: CHANGES.txt
===================================================================
--- CHANGES.txt (revision 737779)
+++ CHANGES.txt (working copy)
@@ -1,4 +1,4 @@
-Lucene Change Log
+Lucene Change Log
$Id$
======================= Trunk (not yet released) =======================
@@ -128,6 +128,13 @@
specify a custom FieldCache parser to generate numeric values from
terms for a field. (Uwe Schindler via Mike McCandless)
+11. LUCENE-1314: Add clone(), clone(boolean readOnly) and
+ reopen(boolean readOnly) to IndexReader. Cloning an IndexReader
+ gives you a new reader which you can make changes to (deletions,
+ norms) without affecting the original reader. Now, with clone or
+ reopen you can change the readOnly of the original reader. (Jason
+ Rutherglen, Mike McCandless)
+
Optimizations
1. LUCENE-1427: Fixed QueryWrapperFilter to not waste time computing
Index: src/test/org/apache/lucene/index/TestIndexReaderClone.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexReaderClone.java (revision 0)
+++ src/test/org/apache/lucene/index/TestIndexReaderClone.java (revision 0)
@@ -0,0 +1,406 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.SegmentReader.Norm;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Tests cloning multiple types of readers, modifying the deletedDocs and norms
+ * and verifies copy on write semantics of the deletedDocs and norms is
+ * implemented properly
+ */
+public class TestIndexReaderClone extends LuceneTestCase {
+
+ public void testCloneReadOnlySegmentReader() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, false);
+ IndexReader reader = IndexReader.open(dir1);
+ IndexReader readOnlyReader = (IndexReader) reader.clone(true);
+ if (!isReadOnly(readOnlyReader)) {
+ fail("reader isn't read only");
+ }
+ if (deleteWorked(1, readOnlyReader)) {
+ fail("deleting from the original should not have worked");
+ }
+ reader.close();
+ readOnlyReader.close();
+ dir1.close();
+ }
+
+ // open non-readOnly reader1, clone to non-readOnly
+ // reader2, make sure we can change reader2
+ public void testCloneNoChangesStillReadOnly() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader r1 = IndexReader.open(dir1, false);
+ DirectoryIndexReader r2 = (DirectoryIndexReader) r1.clone(false);
+ if (!deleteWorked(1, r2)) {
+ fail("deleting from the cloned should have worked");
+ }
+ r1.close();
+ r2.close();
+ dir1.close();
+ }
+
+ // open non-readOnly reader1, clone to non-readOnly
+ // reader2, make sure we can change reader1
+ public void testCloneWriteToOrig() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader r1 = IndexReader.open(dir1, false);
+ DirectoryIndexReader r2 = (DirectoryIndexReader) r1.clone(false);
+ if (!deleteWorked(1, r1)) {
+ fail("deleting from the original should have worked");
+ }
+ r1.close();
+ r2.close();
+ dir1.close();
+ }
+
+ // open non-readOnly reader1, clone to non-readOnly
+ // reader2, make sure we can change reader2
+ public void testCloneWriteToClone() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader r1 = IndexReader.open(dir1, false);
+ DirectoryIndexReader r2 = (DirectoryIndexReader) r1.clone(false);
+ if (!deleteWorked(1, r2)) {
+ fail("deleting from the original should have worked");
+ }
+ // should fail because reader1 holds the write lock
+ assertTrue("first reader should not be able to delete", !deleteWorked(1, r1));
+ r2.close();
+ // should fail because we are now stale (reader1
+ // committed changes)
+ assertTrue("first reader should not be able to delete", !deleteWorked(1, r1));
+ r1.close();
+
+ dir1.close();
+ }
+
+ // create single-segment index, open non-readOnly
+ // SegmentReader, add docs, reopen to multireader, then do
+ // delete
+ public void testReopenSegmentReaderToMultiReader() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, false);
+ IndexReader reader1 = IndexReader.open(dir1, false);
+
+ TestIndexReaderReopen.modifyIndex(5, dir1);
+
+ IndexReader reader2 = reader1.reopen();
+ assertTrue(reader1 != reader2);
+
+ assertTrue(deleteWorked(1, reader2));
+ reader1.close();
+ reader2.close();
+ dir1.close();
+ }
+
+ // open non-readOnly reader1, clone to readOnly reader2
+ public void testCloneWriteableToReadOnly() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader reader = IndexReader.open(dir1, false);
+ DirectoryIndexReader readOnlyReader = (DirectoryIndexReader) reader.clone(true);
+ if (!isReadOnly(readOnlyReader)) {
+ fail("reader isn't read only");
+ }
+ if (deleteWorked(1, readOnlyReader)) {
+ fail("deleting from the original should not have worked");
+ }
+ // this readonly reader shouldn't have a write lock
+ if (readOnlyReader.hasChanges) {
+ fail("readOnlyReader has a write lock");
+ }
+ reader.close();
+ readOnlyReader.close();
+ dir1.close();
+ }
+
+ // open non-readOnly reader1, reopen to readOnly reader2
+ public void testReopenWriteableToReadOnly() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader reader = IndexReader.open(dir1, false);
+ final int docCount = reader.numDocs();
+ assertTrue(deleteWorked(1, reader));
+ assertEquals(docCount-1, reader.numDocs());
+
+ DirectoryIndexReader readOnlyReader = (DirectoryIndexReader) reader.reopen(true);
+ if (!isReadOnly(readOnlyReader)) {
+ fail("reader isn't read only");
+ }
+ assertFalse(deleteWorked(1, readOnlyReader));
+ assertEquals(docCount-1, readOnlyReader.numDocs());
+ reader.close();
+ readOnlyReader.close();
+ dir1.close();
+ }
+
+ // open readOnly reader1, clone to non-readOnly reader2
+ public void testCloneReadOnlyToWriteable() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader reader1 = IndexReader.open(dir1, true);
+
+ DirectoryIndexReader reader2 = (DirectoryIndexReader) reader1.clone(false);
+ if (isReadOnly(reader2)) {
+ fail("reader should not be read only");
+ }
+ assertFalse("deleting from the original reader should not have worked", deleteWorked(1, reader1));
+ // this readonly reader shouldn't yet have a write lock
+ if (reader2.hasChanges) {
+ fail("cloned reader should not have write lock");
+ }
+ assertTrue("deleting from the cloned reader should have worked", deleteWorked(1, reader2));
+ reader1.close();
+ reader2.close();
+ dir1.close();
+ }
+
+ // open non-readOnly reader1 on multi-segment index, then
+ // optimize the index, then clone to readOnly reader2
+ public void testReadOnlyCloneAfterOptimize() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader reader1 = IndexReader.open(dir1, false);
+ IndexWriter w = new IndexWriter(dir1, new SimpleAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
+ w.optimize();
+ w.close();
+ IndexReader reader2 = reader1.clone(true);
+ assertTrue(isReadOnly(reader2));
+ reader1.close();
+ reader2.close();
+ dir1.close();
+ }
+
+ private static boolean deleteWorked(int doc, IndexReader r) {
+ boolean exception = false;
+ try {
+ // trying to delete from the original reader should throw an exception
+ r.deleteDocument(doc);
+ } catch (Exception ex) {
+ exception = true;
+ }
+ return !exception;
+ }
+
+ public void testCloneReadOnlyMultiSegmentReader() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+
+ TestIndexReaderReopen.createIndex(dir1, true);
+ IndexReader reader = IndexReader.open(dir1);
+ IndexReader readOnlyReader = (IndexReader) reader.clone(true);
+ if (!isReadOnly(readOnlyReader)) {
+ fail("reader isn't read only");
+ }
+ reader.close();
+ readOnlyReader.close();
+ dir1.close();
+ }
+
+ public static boolean isReadOnly(IndexReader r) {
+ if (r instanceof ReadOnlySegmentReader
+ || r instanceof ReadOnlyMultiSegmentReader)
+ return true;
+ return false;
+ }
+
+ public void testParallelReader() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir1, true);
+ final Directory dir2 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir2, true);
+ IndexReader r1 = IndexReader.open(dir1);
+ IndexReader r2 = IndexReader.open(dir2);
+
+ ParallelReader pr1 = new ParallelReader();
+ pr1.add(r1);
+ pr1.add(r2);
+
+ performDefaultTests(pr1);
+ pr1.close();
+ dir1.close();
+ dir2.close();
+ }
+
+ /**
+ * 1. Get a norm from the original reader 2. Clone the original reader 3.
+ * Delete a document and set the norm of the cloned reader 4. Verify the norms
+ * are not the same on each reader 5. Verify the doc deleted is only in the
+ * cloned reader 6. Try to delete a document in the original reader, an
+ * exception should be thrown
+ *
+ * @param r1 IndexReader to perform tests on
+ * @throws Exception
+ */
+ private void performDefaultTests(IndexReader r1) throws Exception {
+ float norm1 = Similarity.decodeNorm(r1.norms("field1")[4]);
+
+ IndexReader pr1Clone = (IndexReader) r1.clone();
+ pr1Clone.deleteDocument(10);
+ pr1Clone.setNorm(4, "field1", 0.5f);
+ assertTrue(Similarity.decodeNorm(r1.norms("field1")[4]) == norm1);
+ assertTrue(Similarity.decodeNorm(pr1Clone.norms("field1")[4]) != norm1);
+
+ assertTrue(!r1.isDeleted(10));
+ assertTrue(pr1Clone.isDeleted(10));
+
+ // try to update the original reader, which should throw an exception
+ try {
+ r1.deleteDocument(11);
+ fail("Tried to delete doc 11 and an exception should have been thrown");
+ } catch (Exception exception) {
+ // expectted
+ }
+ pr1Clone.close();
+ }
+
+ public void testMixedReaders() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir1, true);
+ final Directory dir2 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir2, true);
+ IndexReader r1 = IndexReader.open(dir1);
+ IndexReader r2 = IndexReader.open(dir2);
+
+ MultiReader multiReader = new MultiReader(new IndexReader[] { r1, r2 });
+ performDefaultTests(multiReader);
+ multiReader.close();
+ dir1.close();
+ dir2.close();
+ }
+
+ public void testSegmentReaderUndeleteall() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir1, false);
+ SegmentReader origSegmentReader = (SegmentReader) IndexReader.open(dir1);
+ origSegmentReader.deleteDocument(10);
+ assertDelDocsRefCountEquals(1, origSegmentReader);
+ origSegmentReader.undeleteAll();
+ assertNull(origSegmentReader.deletedDocsRef);
+ origSegmentReader.close();
+ // need to test norms?
+ dir1.close();
+ }
+
+ public void testSegmentReaderCloseReferencing() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir1, false);
+ SegmentReader origSegmentReader = (SegmentReader) IndexReader.open(dir1);
+ origSegmentReader.deleteDocument(1);
+ origSegmentReader.setNorm(4, "field1", 0.5f);
+
+ SegmentReader clonedSegmentReader = (SegmentReader) origSegmentReader
+ .clone();
+ assertDelDocsRefCountEquals(2, origSegmentReader);
+ origSegmentReader.close();
+ assertDelDocsRefCountEquals(1, origSegmentReader);
+ // check the norm refs
+ Norm norm = (Norm) clonedSegmentReader.norms.get("field1");
+ assertEquals(1, norm.bytesRef().refCount());
+ clonedSegmentReader.close();
+ dir1.close();
+ }
+
+ public void testSegmentReaderDelDocsReferenceCounting() throws Exception {
+ final Directory dir1 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir1, false);
+ SegmentReader origSegmentReader = (SegmentReader) IndexReader.open(dir1);
+ // deletedDocsRef should be null because nothing has updated yet
+ assertNull(origSegmentReader.deletedDocsRef);
+
+ // we deleted a document, so there is now a deletedDocs bitvector and a
+ // reference to it
+ origSegmentReader.deleteDocument(1);
+ assertDelDocsRefCountEquals(1, origSegmentReader);
+
+ // the cloned segmentreader should have 2 references, 1 to itself, and 1 to
+ // the original segmentreader
+ SegmentReader clonedSegmentReader = (SegmentReader) origSegmentReader
+ .clone();
+ assertDelDocsRefCountEquals(2, origSegmentReader);
+ // deleting a document creates a new deletedDocs bitvector, the refs goes to
+ // 1
+ clonedSegmentReader.deleteDocument(2);
+ assertDelDocsRefCountEquals(1, origSegmentReader);
+ assertDelDocsRefCountEquals(1, clonedSegmentReader);
+
+ // make sure the deletedocs objects are different (copy
+ // on write)
+ assertTrue(origSegmentReader.deletedDocs != clonedSegmentReader.deletedDocs);
+
+ assertDocDeleted(origSegmentReader, clonedSegmentReader, 1);
+ assertTrue(!origSegmentReader.isDeleted(2)); // doc 2 should not be deleted
+ // in original segmentreader
+ assertTrue(clonedSegmentReader.isDeleted(2)); // doc 2 should be deleted in
+ // cloned segmentreader
+
+ // deleting a doc from the original segmentreader should throw an exception
+ try {
+ origSegmentReader.deleteDocument(4);
+ fail("expected exception");
+ } catch (LockObtainFailedException lbfe) {
+ // expected
+ }
+
+ origSegmentReader.close();
+ // try closing the original segment reader to see if it affects the
+ // clonedSegmentReader
+ clonedSegmentReader.deleteDocument(3);
+ clonedSegmentReader.flush();
+ assertDelDocsRefCountEquals(1, clonedSegmentReader);
+
+ // test a reopened reader
+ SegmentReader reopenedSegmentReader = (SegmentReader) clonedSegmentReader
+ .reopen();
+ SegmentReader cloneSegmentReader2 = (SegmentReader) reopenedSegmentReader
+ .clone();
+ assertDelDocsRefCountEquals(2, cloneSegmentReader2);
+ clonedSegmentReader.close();
+ reopenedSegmentReader.close();
+ cloneSegmentReader2.close();
+
+ dir1.close();
+ }
+
+ private void assertDocDeleted(SegmentReader reader, SegmentReader reader2,
+ int doc) {
+ assertEquals(reader.isDeleted(doc), reader2.isDeleted(doc));
+ }
+
+ private void assertDelDocsRefCountEquals(int refCount, SegmentReader reader) {
+ assertEquals(refCount, reader.deletedDocsRef.refCount());
+ }
+}
Property changes on: src/test/org/apache/lucene/index/TestIndexReaderClone.java
___________________________________________________________________
Added: svn:eol-style
+ native
Index: src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision 0)
+++ src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision 0)
@@ -0,0 +1,333 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.SegmentReader.Norm;
+import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Tests cloning IndexReader norms
+ */
+public class TestIndexReaderCloneNorms extends LuceneTestCase {
+
+ private class SimilarityOne extends DefaultSimilarity {
+ public float lengthNorm(String fieldName, int numTerms) {
+ return 1;
+ }
+ }
+
+ private static final int NUM_FIELDS = 10;
+
+ private Similarity similarityOne;
+
+ private Analyzer anlzr;
+
+ private int numDocNorms;
+
+ private ArrayList norms;
+
+ private ArrayList modifiedNorms;
+
+ private float lastNorm = 0;
+
+ private float normDelta = (float) 0.001;
+
+ public TestIndexReaderCloneNorms(String s) {
+ super(s);
+ }
+
+ protected void setUp() throws Exception {
+ super.setUp();
+ similarityOne = new SimilarityOne();
+ anlzr = new StandardAnalyzer();
+ }
+
+ /**
+ * Test that norms values are preserved as the index is maintained. Including
+ * separate norms. Including merging indexes with seprate norms. Including
+ * optimize.
+ */
+ public void testNorms() throws IOException {
+ // tmp dir
+ String tempDir = System.getProperty("java.io.tmpdir");
+ if (tempDir == null) {
+ throw new IOException("java.io.tmpdir undefined, cannot run test");
+ }
+
+ // test with a single index: index1
+ File indexDir1 = new File(tempDir, "lucenetestindex1");
+ Directory dir1 = FSDirectory.getDirectory(indexDir1);
+ IndexWriter.unlock(dir1);
+
+ norms = new ArrayList();
+ modifiedNorms = new ArrayList();
+
+ createIndex(dir1);
+ doTestNorms(dir1);
+
+ // test with a single index: index2
+ ArrayList norms1 = norms;
+ ArrayList modifiedNorms1 = modifiedNorms;
+ int numDocNorms1 = numDocNorms;
+
+ norms = new ArrayList();
+ modifiedNorms = new ArrayList();
+ numDocNorms = 0;
+
+ File indexDir2 = new File(tempDir, "lucenetestindex2");
+ Directory dir2 = FSDirectory.getDirectory(indexDir2);
+
+ createIndex(dir2);
+ doTestNorms(dir2);
+
+ // add index1 and index2 to a third index: index3
+ File indexDir3 = new File(tempDir, "lucenetestindex3");
+ Directory dir3 = FSDirectory.getDirectory(indexDir3);
+
+ createIndex(dir3);
+ IndexWriter iw = new IndexWriter(dir3, anlzr, false,
+ IndexWriter.MaxFieldLength.LIMITED);
+ iw.setMaxBufferedDocs(5);
+ iw.setMergeFactor(3);
+ iw.addIndexes(new Directory[] { dir1, dir2 });
+ iw.close();
+
+ norms1.addAll(norms);
+ norms = norms1;
+ modifiedNorms1.addAll(modifiedNorms);
+ modifiedNorms = modifiedNorms1;
+ numDocNorms += numDocNorms1;
+
+ // test with index3
+ verifyIndex(dir3);
+ doTestNorms(dir3);
+
+ // now with optimize
+ iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
+ iw.setMaxBufferedDocs(5);
+ iw.setMergeFactor(3);
+ iw.optimize();
+ iw.close();
+ verifyIndex(dir3);
+
+ dir1.close();
+ dir2.close();
+ dir3.close();
+ }
+
+ // try cloning and reopening the norms
+ private void doTestNorms(Directory dir) throws IOException {
+ addDocs(dir, 12, true);
+ IndexReader ir = IndexReader.open(dir);
+ verifyIndex(ir);
+ modifyNormsForF1(ir);
+ IndexReader irc = (IndexReader) ir.clone();// IndexReader.open(dir);//ir.clone();
+ verifyIndex(irc);
+
+ modifyNormsForF1(irc);
+
+ IndexReader irc3 = (IndexReader) irc.clone();
+ verifyIndex(irc3);
+ modifyNormsForF1(irc3);
+ verifyIndex(irc3);
+ irc3.flush();
+ irc3.close();
+ }
+
+ public void testNormsClose() throws IOException {
+ Directory dir1 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir1, false);
+ SegmentReader reader1 = (SegmentReader) IndexReader.open(dir1);
+ reader1.norms("field1");
+ Norm r1norm = (Norm)reader1.norms.get("field1");
+ SegmentReader.Ref r1BytesRef = r1norm.bytesRef();
+ SegmentReader reader2 = (SegmentReader)reader1.clone();
+ assertEquals(2, r1norm.bytesRef().refCount());
+ reader1.close();
+ assertEquals(1, r1BytesRef.refCount());
+ reader2.norms("field1");
+ reader2.close();
+ dir1.close();
+ }
+
+ public void testNormsRefCounting() throws IOException {
+ Directory dir1 = new MockRAMDirectory();
+ TestIndexReaderReopen.createIndex(dir1, false);
+ SegmentReader reader1 = (SegmentReader) IndexReader.open(dir1);
+
+ SegmentReader reader2C = (SegmentReader)reader1.clone();
+ reader2C.norms("field1"); // load the norms for the field
+ Norm reader2CNorm = (Norm)reader2C.norms.get("field1");
+ assertTrue("reader2CNorm.bytesRef()=" + reader2CNorm.bytesRef(), reader2CNorm.bytesRef().refCount() == 2);
+
+
+
+ SegmentReader reader3C = (SegmentReader)reader2C.clone();
+ Norm reader3CCNorm = (Norm)reader3C.norms.get("field1");
+ assertEquals(3, reader3CCNorm.bytesRef().refCount());
+
+ // edit a norm and the refcount should be 1
+ SegmentReader reader4C = (SegmentReader)reader3C.clone();
+ assertEquals(4, reader3CCNorm.bytesRef().refCount());
+ reader4C.setNorm(5, "field1", 0.33f);
+
+ // generate a cannot update exception in reader1
+ try {
+ reader3C.setNorm(1, "field1", 0.99f);
+ fail("did not hit expected exception");
+ } catch (Exception ex) {
+ // expected
+ }
+
+ // norm values should be different
+ assertTrue(Similarity.decodeNorm(reader3C.norms("field1")[5]) != Similarity.decodeNorm(reader4C.norms("field1")[5]));
+ Norm reader4CCNorm = (Norm)reader4C.norms.get("field1");
+ assertEquals(3, reader3CCNorm.bytesRef().refCount());
+ assertEquals(1, reader4CCNorm.bytesRef().refCount());
+
+ SegmentReader reader5C = (SegmentReader)reader4C.clone();
+ Norm reader5CCNorm = (Norm)reader5C.norms.get("field1");
+ reader5C.setNorm(5, "field1", 0.7f);
+ assertEquals(1, reader5CCNorm.bytesRef().refCount());
+
+ reader5C.close();
+ reader4C.close();
+ reader3C.close();
+ reader2C.close();
+ reader1.close();
+ dir1.close();
+ }
+
+ private void createIndex(Directory dir) throws IOException {
+ IndexWriter iw = new IndexWriter(dir, anlzr, true,
+ IndexWriter.MaxFieldLength.LIMITED);
+ iw.setMaxBufferedDocs(5);
+ iw.setMergeFactor(3);
+ iw.setSimilarity(similarityOne);
+ iw.setUseCompoundFile(true);
+ iw.close();
+ }
+
+ private void modifyNormsForF1(Directory dir) throws IOException {
+ IndexReader ir = IndexReader.open(dir);
+ modifyNormsForF1(ir);
+ }
+
+ private void modifyNormsForF1(IndexReader ir) throws IOException {
+ int n = ir.maxDoc();
+ // System.out.println("modifyNormsForF1 maxDoc: "+n);
+ for (int i = 0; i < n; i += 3) { // modify for every third doc
+ int k = (i * 3) % modifiedNorms.size();
+ float origNorm = ((Float) modifiedNorms.get(i)).floatValue();
+ float newNorm = ((Float) modifiedNorms.get(k)).floatValue();
+ // System.out.println("Modifying: for "+i+" from "+origNorm+" to
+ // "+newNorm);
+ // System.out.println(" and: for "+k+" from "+newNorm+" to "+origNorm);
+ modifiedNorms.set(i, new Float(newNorm));
+ modifiedNorms.set(k, new Float(origNorm));
+ ir.setNorm(i, "f" + 1, newNorm);
+ ir.setNorm(k, "f" + 1, origNorm);
+ // System.out.println("setNorm i: "+i);
+ // break;
+ }
+ // ir.close();
+ }
+
+ private void verifyIndex(Directory dir) throws IOException {
+ IndexReader ir = IndexReader.open(dir);
+ verifyIndex(ir);
+ ir.close();
+ }
+
+ private void verifyIndex(IndexReader ir) throws IOException {
+ for (int i = 0; i < NUM_FIELDS; i++) {
+ String field = "f" + i;
+ byte b[] = ir.norms(field);
+ assertEquals("number of norms mismatches", numDocNorms, b.length);
+ ArrayList storedNorms = (i == 1 ? modifiedNorms : norms);
+ for (int j = 0; j < b.length; j++) {
+ float norm = Similarity.decodeNorm(b[j]);
+ float norm1 = ((Float) storedNorms.get(j)).floatValue();
+ assertEquals("stored norm value of " + field + " for doc " + j + " is "
+ + norm + " - a mismatch!", norm, norm1, 0.000001);
+ }
+ }
+ }
+
+ private void addDocs(Directory dir, int ndocs, boolean compound)
+ throws IOException {
+ IndexWriter iw = new IndexWriter(dir, anlzr, false,
+ IndexWriter.MaxFieldLength.LIMITED);
+ iw.setMaxBufferedDocs(5);
+ iw.setMergeFactor(3);
+ iw.setSimilarity(similarityOne);
+ iw.setUseCompoundFile(compound);
+ for (int i = 0; i < ndocs; i++) {
+ iw.addDocument(newDoc());
+ }
+ iw.close();
+ }
+
+ // create the next document
+ private Document newDoc() {
+ Document d = new Document();
+ float boost = nextNorm();
+ for (int i = 0; i < 10; i++) {
+ Field f = new Field("f" + i, "v" + i, Store.NO, Index.NOT_ANALYZED);
+ f.setBoost(boost);
+ d.add(f);
+ }
+ return d;
+ }
+
+ // return unique norm values that are unchanged by encoding/decoding
+ private float nextNorm() {
+ float norm = lastNorm + normDelta;
+ do {
+ float norm1 = Similarity.decodeNorm(Similarity.encodeNorm(norm));
+ if (norm1 > lastNorm) {
+ // System.out.println(norm1+" > "+lastNorm);
+ norm = norm1;
+ break;
+ }
+ norm += normDelta;
+ } while (true);
+ norms.add(numDocNorms, new Float(norm));
+ modifiedNorms.add(numDocNorms, new Float(norm));
+ // System.out.println("creating norm("+numDocNorms+"): "+norm);
+ numDocNorms++;
+ lastNorm = (norm > 10 ? 0 : norm); // there's a limit to how many distinct
+ // values can be stored in a ingle byte
+ return norm;
+ }
+}
Property changes on: src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
___________________________________________________________________
Added: svn:eol-style
+ native
Index: src/test/org/apache/lucene/index/TestIndexReaderReopen.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 737779)
+++ src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy)
@@ -40,16 +40,16 @@
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.MockRAMDirectory;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.util.LuceneTestCase;
public class TestIndexReaderReopen extends LuceneTestCase {
private File indexDir;
-
+
public void testReopen() throws Exception {
- final Directory dir1 = new RAMDirectory();
+ final Directory dir1 = new MockRAMDirectory();
createIndex(dir1, false);
performDefaultTests(new TestReopen() {
@@ -63,8 +63,9 @@
}
});
+ dir1.close();
- final Directory dir2 = new RAMDirectory();
+ final Directory dir2 = new MockRAMDirectory();
createIndex(dir2, true);
performDefaultTests(new TestReopen() {
@@ -78,12 +79,13 @@
}
});
+ dir2.close();
}
public void testParallelReaderReopen() throws Exception {
- final Directory dir1 = new RAMDirectory();
+ final Directory dir1 = new MockRAMDirectory();
createIndex(dir1, true);
- final Directory dir2 = new RAMDirectory();
+ final Directory dir2 = new MockRAMDirectory();
createIndex(dir2, true);
performDefaultTests(new TestReopen() {
@@ -101,10 +103,12 @@
}
});
+ dir1.close();
+ dir2.close();
- final Directory dir3 = new RAMDirectory();
+ final Directory dir3 = new MockRAMDirectory();
createIndex(dir3, true);
- final Directory dir4 = new RAMDirectory();
+ final Directory dir4 = new MockRAMDirectory();
createIndex(dir4, true);
performTestsWithExceptionInReopen(new TestReopen() {
@@ -118,11 +122,15 @@
ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir3));
pr.add(IndexReader.open(dir4));
+ // Does not implement reopen, so
+ // hits exception:
pr.add(new FilterIndexReader(IndexReader.open(dir3)));
return pr;
}
});
+ dir3.close();
+ dir4.close();
}
// LUCENE-1228: IndexWriter.commit() does not update the index version
@@ -133,17 +141,20 @@
public void testCommitReopenFS () throws IOException {
Directory dir = FSDirectory.getDirectory(indexDir);
doTestReopenWithCommit(dir, true);
+ dir.close();
}
public void testCommitRecreateFS () throws IOException {
Directory dir = FSDirectory.getDirectory(indexDir);
doTestReopenWithCommit(dir, false);
+ dir.close();
}
public void testCommitReopenRAM () throws IOException {
- Directory dir = new RAMDirectory();
+ Directory dir = new MockRAMDirectory();
doTestReopenWithCommit(dir, true);
+ dir.close();
}
public void testCommitRecreateRAM () throws IOException {
- Directory dir = new RAMDirectory();
+ Directory dir = new MockRAMDirectory();
doTestReopenWithCommit(dir, false);
}
@@ -188,9 +199,11 @@
}
public void testMultiReaderReopen() throws Exception {
- final Directory dir1 = new RAMDirectory();
+
+ final Directory dir1 = new MockRAMDirectory();
createIndex(dir1, true);
- final Directory dir2 = new RAMDirectory();
+
+ final Directory dir2 = new MockRAMDirectory();
createIndex(dir2, true);
performDefaultTests(new TestReopen() {
@@ -207,10 +220,14 @@
}
});
+
+ dir1.close();
+ dir2.close();
- final Directory dir3 = new RAMDirectory();
+ final Directory dir3 = new MockRAMDirectory();
createIndex(dir3, true);
- final Directory dir4 = new RAMDirectory();
+
+ final Directory dir4 = new MockRAMDirectory();
createIndex(dir4, true);
performTestsWithExceptionInReopen(new TestReopen() {
@@ -224,23 +241,26 @@
return new MultiReader(new IndexReader[]
{IndexReader.open(dir3),
IndexReader.open(dir4),
+ // Does not implement reopen, so
+ // hits exception:
new FilterIndexReader(IndexReader.open(dir3))});
}
});
-
+ dir3.close();
+ dir4.close();
}
public void testMixedReaders() throws Exception {
- final Directory dir1 = new RAMDirectory();
+ final Directory dir1 = new MockRAMDirectory();
createIndex(dir1, true);
- final Directory dir2 = new RAMDirectory();
+ final Directory dir2 = new MockRAMDirectory();
createIndex(dir2, true);
- final Directory dir3 = new RAMDirectory();
+ final Directory dir3 = new MockRAMDirectory();
createIndex(dir3, false);
- final Directory dir4 = new RAMDirectory();
+ final Directory dir4 = new MockRAMDirectory();
createIndex(dir4, true);
- final Directory dir5 = new RAMDirectory();
+ final Directory dir5 = new MockRAMDirectory();
createIndex(dir5, false);
performDefaultTests(new TestReopen() {
@@ -263,33 +283,39 @@
pr, mr, IndexReader.open(dir5)});
}
});
+ dir1.close();
+ dir2.close();
+ dir3.close();
+ dir4.close();
+ dir5.close();
}
-
private void performDefaultTests(TestReopen test) throws Exception {
+
IndexReader index1 = test.openReader();
IndexReader index2 = test.openReader();
TestIndexReader.assertIndexEquals(index1, index2);
-
+
// verify that reopen() does not return a new reader instance
// in case the index has no changes
ReaderCouple couple = refreshReader(index2, false);
assertTrue(couple.refreshedReader == index2);
couple = refreshReader(index2, test, 0, true);
+ index1.close();
index1 = couple.newReader;
+
IndexReader index2_refreshed = couple.refreshedReader;
index2.close();
// test if refreshed reader and newly opened reader return equal results
TestIndexReader.assertIndexEquals(index1, index2_refreshed);
-
- index1.close();
+
index2_refreshed.close();
assertReaderClosed(index2, true, true);
assertReaderClosed(index2_refreshed, true, true);
-
+
index2 = test.openReader();
for (int i = 1; i < 4; i++) {
@@ -313,7 +339,7 @@
public void testReferenceCounting() throws IOException {
for (int mode = 0; mode < 4; mode++) {
- Directory dir1 = new RAMDirectory();
+ Directory dir1 = new MockRAMDirectory();
createIndex(dir1, true);
IndexReader reader0 = IndexReader.open(dir1);
@@ -408,21 +434,24 @@
assertReaderClosed(reader1, true, true);
assertReaderClosed(reader2, true, true);
assertReaderClosed(reader3, true, true);
+
+ dir1.close();
}
}
public void testReferenceCountingMultiReader() throws IOException {
for (int mode = 0; mode <=1; mode++) {
- Directory dir1 = new RAMDirectory();
+ Directory dir1 = new MockRAMDirectory();
createIndex(dir1, false);
- Directory dir2 = new RAMDirectory();
+ Directory dir2 = new MockRAMDirectory();
createIndex(dir2, true);
IndexReader reader1 = IndexReader.open(dir1);
assertRefCountEquals(1, reader1);
-
- IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, IndexReader.open(dir2)}, (mode == 0));
+
+ IndexReader initReader2 = IndexReader.open(dir2);
+ IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, initReader2}, (mode == 0));
modifyIndex(0, dir2);
assertRefCountEquals(1 + mode, reader1);
@@ -433,6 +462,10 @@
modifyIndex(0, dir1);
IndexReader reader2 = reader1.reopen();
assertRefCountEquals(3 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
modifyIndex(1, dir1);
IndexReader reader3 = reader2.reopen();
@@ -444,6 +477,10 @@
multiReader1.close();
assertRefCountEquals(3 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
reader1.close();
assertRefCountEquals(3, reader1);
@@ -468,15 +505,17 @@
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, true);
+ dir1.close();
+ dir2.close();
}
}
public void testReferenceCountingParallelReader() throws IOException {
for (int mode = 0; mode <=1; mode++) {
- Directory dir1 = new RAMDirectory();
+ Directory dir1 = new MockRAMDirectory();
createIndex(dir1, false);
- Directory dir2 = new RAMDirectory();
+ Directory dir2 = new MockRAMDirectory();
createIndex(dir2, true);
IndexReader reader1 = IndexReader.open(dir1);
@@ -484,7 +523,8 @@
ParallelReader parallelReader1 = new ParallelReader(mode == 0);
parallelReader1.add(reader1);
- parallelReader1.add(IndexReader.open(dir2));
+ IndexReader initReader2 = IndexReader.open(dir2);
+ parallelReader1.add(initReader2);
modifyIndex(1, dir2);
assertRefCountEquals(1 + mode, reader1);
@@ -496,6 +536,10 @@
modifyIndex(0, dir2);
IndexReader reader2 = reader1.reopen();
assertRefCountEquals(3 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
modifyIndex(4, dir1);
IndexReader reader3 = reader2.reopen();
@@ -507,6 +551,10 @@
parallelReader1.close();
assertRefCountEquals(3 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
reader1.close();
assertRefCountEquals(3, reader1);
@@ -531,12 +579,15 @@
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, true);
+
+ dir1.close();
+ dir2.close();
}
}
public void testNormsRefCounting() throws IOException {
- Directory dir1 = new RAMDirectory();
+ Directory dir1 = new MockRAMDirectory();
createIndex(dir1, false);
SegmentReader reader1 = (SegmentReader) IndexReader.open(dir1);
@@ -554,25 +605,31 @@
modifier = IndexReader.open(dir1);
modifier.deleteDocument(2);
modifier.close();
+
SegmentReader reader4 = (SegmentReader) reader3.reopen();
-
modifier = IndexReader.open(dir1);
modifier.deleteDocument(3);
modifier.close();
+
SegmentReader reader5 = (SegmentReader) reader3.reopen();
// Now reader2-reader5 references reader1. reader1 and reader2
// share the same norms. reader3, reader4, reader5 also share norms.
assertRefCountEquals(5, reader1);
assertFalse(reader1.normsClosed());
+
reader1.close();
+
assertRefCountEquals(4, reader1);
assertFalse(reader1.normsClosed());
+
reader2.close();
assertRefCountEquals(3, reader1);
+
// now the norms for field1 and field2 should be closed
assertTrue(reader1.normsClosed("field1"));
assertTrue(reader1.normsClosed("field2"));
+
// but the norms for field3 and field4 should still be open
assertFalse(reader1.normsClosed("field3"));
assertFalse(reader1.normsClosed("field4"));
@@ -593,6 +650,8 @@
// the norms that those three readers shared should be
// closed as well
assertTrue(reader3.normsClosed());
+
+ dir1.close();
}
private void performTestsWithExceptionInReopen(TestReopen test) throws Exception {
@@ -610,10 +669,13 @@
// index2 should still be usable and unaffected by the failed reopen() call
TestIndexReader.assertIndexEquals(index1, index2);
+
+ index1.close();
+ index2.close();
}
public void testThreadSafety() throws Exception {
- final Directory dir = new RAMDirectory();
+ final Directory dir = new MockRAMDirectory();
final int n = 150;
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
@@ -779,6 +841,8 @@
assertReaderClosed(reader, true, true);
assertReaderClosed(firstReader, true, true);
+
+ dir.close();
}
private static class ReaderCouple {
@@ -829,7 +893,7 @@
return refreshReader(reader, null, -1, hasChanges);
}
- private ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException {
+ ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException {
synchronized (createReaderMutex) {
IndexReader r = null;
if (test != null) {
@@ -837,7 +901,16 @@
r = test.openReader();
}
- IndexReader refreshed = reader.reopen();
+ IndexReader refreshed = null;
+ try {
+ refreshed = reader.reopen();
+ } finally {
+ if (refreshed == null && r != null) {
+ // Hit exception -- close opened reader
+ r.close();
+ }
+ }
+
if (hasChanges) {
if (refreshed == reader) {
fail("No new IndexReader instance created during refresh.");
@@ -852,9 +925,10 @@
}
}
- private static void createIndex(Directory dir, boolean multiSegment) throws IOException {
+ public static void createIndex(Directory dir, boolean multiSegment) throws IOException {
+ IndexWriter.unlock(dir);
IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
-
+
w.setMergePolicy(new LogDocMergePolicy());
for (int i = 0; i < 100; i++) {
@@ -893,7 +967,7 @@
return doc;
}
- private static void modifyIndex(int i, Directory dir) throws IOException {
+ static void modifyIndex(int i, Directory dir) throws IOException {
switch (i) {
case 0: {
IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
@@ -932,7 +1006,12 @@
reader.close();
break;
}
-
+ case 5: {
+ IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
+ w.addDocument(createDocument(101, 4));
+ w.close();
+ break;
+ }
}
}
Index: src/java/org/apache/lucene/index/DirectoryIndexReader.java
===================================================================
--- src/java/org/apache/lucene/index/DirectoryIndexReader.java (revision 737779)
+++ src/java/org/apache/lucene/index/DirectoryIndexReader.java (working copy)
@@ -37,7 +37,7 @@
* "own" the directory, which means that they try to acquire a write lock
* whenever index modifications are performed.
*/
-abstract class DirectoryIndexReader extends IndexReader {
+abstract class DirectoryIndexReader extends IndexReader implements Cloneable {
protected Directory directory;
protected boolean closeDirectory;
private IndexDeletionPolicy deletionPolicy;
@@ -62,12 +62,24 @@
this.closeDirectory = closeDirectory;
this.readOnly = readOnly;
+ if (readOnly) {
+ assert this instanceof ReadOnlySegmentReader ||
+ this instanceof ReadOnlyMultiSegmentReader;
+ } else {
+ assert !(this instanceof ReadOnlySegmentReader) &&
+ !(this instanceof ReadOnlyMultiSegmentReader);
+ }
+
if (!readOnly && segmentInfos != null) {
// We assume that this segments_N was previously
// properly sync'd:
synced.addAll(segmentInfos.files(directory, true));
}
}
+
+ boolean hasSegmentInfos() {
+ return segmentInfos != null;
+ }
protected DirectoryIndexReader() {}
@@ -134,14 +146,73 @@
return reader;
}
+
+ public final synchronized IndexReader reopen() throws CorruptIndexException, IOException {
+ // Preserve current readOnly
+ return doReopen(readOnly);
+ }
- public final synchronized IndexReader reopen() throws CorruptIndexException, IOException {
+ public final synchronized IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
+ return doReopen(openReadOnly);
+ }
+
+ public final synchronized Object clone() {
+ try {
+ // Preserve current readOnly
+ return clone(readOnly);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ public final synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
+
+ final SegmentInfos infos = (SegmentInfos) segmentInfos.clone();
+ DirectoryIndexReader newReader = doReopen(infos, true, openReadOnly);
+
+ if (this != newReader) {
+ newReader.init(directory, infos, closeDirectory, openReadOnly);
+ newReader.deletionPolicy = deletionPolicy;
+ }
+
+ // If we're cloning a non-readOnly reader, move the
+ // writeLock (if there is one) to the new reader:
+ if (!openReadOnly && writeLock != null) {
+ newReader.writeLock = writeLock;
+ writeLock = null;
+ hasChanges = false;
+ }
+
+ return newReader;
+ }
+
+ // If there are no changes to the index, simply return
+ // ourself. If there are changes, load the latest
+ // SegmentInfos and reopen based on that
+ protected final synchronized IndexReader doReopen(final boolean openReadOnly) throws CorruptIndexException, IOException {
ensureOpen();
- if (this.hasChanges || this.isCurrent()) {
- // this has changes, therefore we have the lock and don't need to reopen
- // OR: the index in the directory hasn't changed - nothing to do here
- return this;
+ if (hasChanges) {
+ // We have changes, which means we are not readOnly:
+ assert readOnly == false;
+ // and we hold the write lock:
+ assert writeLock != null;
+ // so no other writer holds the write lock, which
+ // means no changes could have been done to the index:
+ assert isCurrent();
+
+ if (openReadOnly) {
+ return (IndexReader) clone(openReadOnly);
+ } else {
+ return this;
+ }
+ } else if (isCurrent()) {
+ if (openReadOnly != readOnly) {
+ // Just fallback to clone
+ return (IndexReader) clone(openReadOnly);
+ } else {
+ return this;
+ }
}
final SegmentInfos.FindSegmentsFile finder = new SegmentInfos.FindSegmentsFile(directory) {
@@ -149,11 +220,10 @@
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
infos.read(directory, segmentFileName);
-
- DirectoryIndexReader newReader = doReopen(infos);
+ DirectoryIndexReader newReader = doReopen(infos, false, openReadOnly);
if (DirectoryIndexReader.this != newReader) {
- newReader.init(directory, infos, closeDirectory, readOnly);
+ newReader.init(directory, infos, closeDirectory, openReadOnly);
newReader.deletionPolicy = deletionPolicy;
}
@@ -193,7 +263,7 @@
/**
* Re-opens the index using the passed-in SegmentInfos
*/
- protected abstract DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException;
+ protected abstract DirectoryIndexReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException;
public void setDeletionPolicy(IndexDeletionPolicy deletionPolicy) {
this.deletionPolicy = deletionPolicy;
@@ -341,6 +411,14 @@
* @throws IOException if there is a low-level IO error
*/
protected void acquireWriteLock() throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+
+ if (readOnly) {
+ // NOTE: we should not reach this code w/ the core
+ // IndexReader classes; however, an external subclass
+ // of IndexReader could reach this.
+ ReadOnlySegmentReader.noWrite();
+ }
+
if (segmentInfos != null) {
ensureOpen();
if (stale)
Index: src/java/org/apache/lucene/index/ParallelReader.java
===================================================================
--- src/java/org/apache/lucene/index/ParallelReader.java (revision 737779)
+++ src/java/org/apache/lucene/index/ParallelReader.java (working copy)
@@ -122,7 +122,15 @@
}
decrefOnClose.add(Boolean.valueOf(incRefReaders));
}
-
+
+ public synchronized Object clone() {
+ try {
+ return doReopen(true);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
/**
* Tries to reopen the subreaders.
*
@@ -142,63 +150,42 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexReader reopen() throws CorruptIndexException, IOException {
+ public synchronized IndexReader reopen() throws CorruptIndexException, IOException {
+ return doReopen(false);
+ }
+
+ protected IndexReader doReopen(boolean doClone) throws CorruptIndexException, IOException {
ensureOpen();
boolean reopened = false;
List newReaders = new ArrayList();
- List newDecrefOnClose = new ArrayList();
boolean success = false;
try {
-
for (int i = 0; i < readers.size(); i++) {
IndexReader oldReader = (IndexReader) readers.get(i);
- IndexReader newReader = oldReader.reopen();
+ IndexReader newReader = null;
+ if (doClone) {
+ newReader = (IndexReader) oldReader.clone();
+ } else {
+ newReader = oldReader.reopen();
+ }
newReaders.add(newReader);
// if at least one of the subreaders was updated we remember that
- // and return a new MultiReader
+ // and return a new ParallelReader
if (newReader != oldReader) {
reopened = true;
}
}
-
- if (reopened) {
- ParallelReader pr = new ParallelReader();
- for (int i = 0; i < readers.size(); i++) {
- IndexReader oldReader = (IndexReader) readers.get(i);
- IndexReader newReader = (IndexReader) newReaders.get(i);
- if (newReader == oldReader) {
- newDecrefOnClose.add(Boolean.TRUE);
- newReader.incRef();
- } else {
- // this is a new subreader instance, so on close() we don't
- // decRef but close it
- newDecrefOnClose.add(Boolean.FALSE);
- }
- pr.add(newReader, !storedFieldReaders.contains(oldReader));
- }
- pr.decrefOnClose = newDecrefOnClose;
- pr.incRefReaders = incRefReaders;
- success = true;
- return pr;
- } else {
- success = true;
- // No subreader was refreshed
- return this;
- }
+ success = true;
} finally {
if (!success && reopened) {
for (int i = 0; i < newReaders.size(); i++) {
IndexReader r = (IndexReader) newReaders.get(i);
- if (r != null) {
+ if (r != readers.get(i)) {
try {
- if (((Boolean) newDecrefOnClose.get(i)).booleanValue()) {
- r.decRef();
- } else {
- r.close();
- }
+ r.close();
} catch (IOException ignore) {
// keep going - we want to clean up as much as possible
}
@@ -206,6 +193,30 @@
}
}
}
+
+ if (reopened) {
+ List newDecrefOnClose = new ArrayList();
+ ParallelReader pr = new ParallelReader();
+ for (int i = 0; i < readers.size(); i++) {
+ IndexReader oldReader = (IndexReader) readers.get(i);
+ IndexReader newReader = (IndexReader) newReaders.get(i);
+ if (newReader == oldReader) {
+ newDecrefOnClose.add(Boolean.TRUE);
+ newReader.incRef();
+ } else {
+ // this is a new subreader instance, so on close() we don't
+ // decRef but close it
+ newDecrefOnClose.add(Boolean.FALSE);
+ }
+ pr.add(newReader, !storedFieldReaders.contains(oldReader));
+ }
+ pr.decrefOnClose = newDecrefOnClose;
+ pr.incRefReaders = incRefReaders;
+ return pr;
+ } else {
+ // No subreader was refreshed
+ return this;
+ }
}
Index: src/java/org/apache/lucene/index/SegmentReader.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentReader.java (revision 737779)
+++ src/java/org/apache/lucene/index/SegmentReader.java (working copy)
@@ -48,12 +48,13 @@
FieldInfos fieldInfos;
private FieldsReader fieldsReaderOrig = null;
-
+ CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal();
TermInfosReader tis;
TermVectorsReader termVectorsReaderOrig = null;
CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal();
BitVector deletedDocs = null;
+ Ref deletedDocsRef = null;
private boolean deletedDocsDirty = false;
private boolean normsDirty = false;
private boolean undeleteAll = false;
@@ -63,13 +64,12 @@
private boolean rollbackNormsDirty = false;
private boolean rollbackUndeleteAll = false;
private int rollbackPendingDeleteCount;
- private boolean readOnly;
-
IndexInput freqStream;
IndexInput proxStream;
// optionally used for the .nrm file shared by multiple norms
private IndexInput singleNormStream;
+ private Ref singleNormRef;
// Compound File Reader when based on a compound file segment
CompoundFileReader cfsReader = null;
@@ -87,43 +87,238 @@
return (FieldsReader) fieldsReaderOrig.clone();
}
}
- CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal();
- private class Norm {
- volatile int refCount;
- boolean useSingleNormStream;
+ static class Ref {
+ private int refCount = 1;
- public synchronized void incRef() {
+ public String toString() {
+ return "refcount: "+refCount;
+ }
+
+ public synchronized int refCount() {
+ return refCount;
+ }
+
+ public synchronized int incRef() {
assert refCount > 0;
refCount++;
+ return refCount;
}
- public synchronized void decRef() throws IOException {
+ public synchronized int decRef() {
assert refCount > 0;
- if (refCount == 1) {
- close();
- }
refCount--;
-
+ return refCount;
}
-
- public Norm(IndexInput in, boolean useSingleNormStream, int number, long normSeek)
- {
- refCount = 1;
- this.in = in;
- this.number = number;
- this.normSeek = normSeek;
- this.useSingleNormStream = useSingleNormStream;
- }
+ }
+
+ /**
+ * Byte[] referencing is used because a new norm object needs
+ * to be created for each clone, and the byte array is all
+ * that is needed for sharing between cloned readers. The
+ * current norm referencing is for sharing between readers
+ * whereas the byte[] referencing is for copy on write which
+ * is independent of reader references (i.e. incRef, decRef).
+ */
+ final class Norm implements Cloneable {
+ private int refCount = 1;
+
+ // If this instance is a clone, the originalNorm
+ // references the Norm that has a real open IndexInput:
+ private Norm origNorm;
+
private IndexInput in;
+ private long normSeek;
+
+ // null until bytes is set
+ private Ref bytesRef;
private byte[] bytes;
private boolean dirty;
private int number;
- private long normSeek;
private boolean rollbackDirty;
+
+ public Norm(IndexInput in, int number, long normSeek) {
+ this.in = in;
+ this.number = number;
+ this.normSeek = normSeek;
+ }
- private void reWrite(SegmentInfo si) throws IOException {
+ public synchronized void incRef() {
+ assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
+ refCount++;
+ }
+
+ private void closeInput() throws IOException {
+ if (in != null) {
+ if (in != singleNormStream) {
+ // It's private to us -- just close it
+ in.close();
+ } else {
+ // We are sharing this with others -- decRef and
+ // maybe close the shared norm stream
+ if (singleNormRef.decRef() == 0) {
+ singleNormStream.close();
+ singleNormStream = null;
+ }
+ }
+
+ in = null;
+ }
+ }
+
+ public synchronized void decRef() throws IOException {
+ assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
+
+ if (--refCount == 0) {
+ if (origNorm != null) {
+ origNorm.decRef();
+ origNorm = null;
+ } else {
+ closeInput();
+ }
+
+ if (bytes != null) {
+ assert bytesRef != null;
+ bytesRef.decRef();
+ bytes = null;
+ bytesRef = null;
+ } else {
+ assert bytesRef == null;
+ }
+ }
+ }
+
+ // Load bytes but do not cache them if they were not
+ // already cached
+ public synchronized void bytes(byte[] bytesOut, int offset, int len) throws IOException {
+ assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
+ if (bytes != null) {
+ // Already cached -- copy from cache:
+ assert len <= maxDoc();
+ System.arraycopy(bytes, 0, bytesOut, offset, len);
+ } else {
+ // Not cached
+ if (origNorm != null) {
+ // Ask origNorm to load
+ origNorm.bytes(bytesOut, offset, len);
+ } else {
+ // We are orig -- read ourselves from disk:
+ synchronized(in) {
+ in.seek(normSeek);
+ in.readBytes(bytesOut, offset, len, false);
+ }
+ }
+ }
+ }
+
+ // Load & cache full bytes array. Returns bytes.
+ public synchronized byte[] bytes() throws IOException {
+ assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
+ if (bytes == null) { // value not yet read
+ assert bytesRef == null;
+ if (origNorm != null) {
+ // Ask origNorm to load so that for a series of
+ // reopened readers we share a single read-only
+ // byte[]
+ bytes = origNorm.bytes();
+ bytesRef = origNorm.bytesRef;
+ bytesRef.incRef();
+
+ // Once we've loaded the bytes we no longer need
+ // origNorm:
+ origNorm.decRef();
+ origNorm = null;
+
+ } else {
+ // We are the origNorm, so load the bytes for real
+ // ourself:
+ final int count = maxDoc();
+ bytes = new byte[count];
+
+ // Since we are orig, in must not be null
+ assert in != null;
+
+ // Read from disk.
+ synchronized(in) {
+ in.seek(normSeek);
+ in.readBytes(bytes, 0, count, false);
+ }
+
+ bytesRef = new Ref();
+ closeInput();
+ }
+ }
+
+ return bytes;
+ }
+
+ // Only for testing
+ Ref bytesRef() {
+ return bytesRef;
+ }
+
+ // Called if we intend to change a norm value. We make a
+ // private copy of bytes if it's shared with others:
+ public synchronized byte[] copyOnWrite() throws IOException {
+ assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
+ bytes();
+ assert bytes != null;
+ assert bytesRef != null;
+ if (bytesRef.refCount() > 1) {
+ // I cannot be the origNorm for another norm
+ // instance if I'm being changed. Ie, only the
+ // "head Norm" can be changed:
+ assert refCount == 1;
+ final Ref oldRef = bytesRef;
+ bytes = cloneNormBytes(bytes);
+ bytesRef = new Ref();
+ oldRef.decRef();
+ }
+ dirty = true;
+ return bytes;
+ }
+
+ // Returns a copy of this Norm instance that shares
+ // IndexInput & bytes with the original one
+ public synchronized Object clone() {
+ assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
+
+ Norm clone;
+ try {
+ clone = (Norm) super.clone();
+ } catch (CloneNotSupportedException cnse) {
+ // Cannot happen
+ throw new RuntimeException("unexpected CloneNotSupportedException", cnse);
+ }
+ clone.refCount = 1;
+
+ if (bytes != null) {
+ assert bytesRef != null;
+ assert origNorm == null;
+
+ // Clone holds a reference to my bytes:
+ clone.bytesRef.incRef();
+ } else {
+ assert bytesRef == null;
+ if (origNorm == null) {
+ // I become the origNorm for the clone:
+ clone.origNorm = this;
+ }
+ clone.origNorm.incRef();
+ }
+
+ // Only the origNorm will actually readBytes from in:
+ clone.in = null;
+
+ return clone;
+ }
+
+ // Flush all pending changes to the next generation
+ // separate norms file.
+ public void reWrite(SegmentInfo si) throws IOException {
+ assert refCount > 0 && (origNorm == null || origNorm.refCount > 0): "refCount=" + refCount + " origNorm=" + origNorm;
+
// NOTE: norms are re-written in regular directory, not cfs
si.advanceNormGen(this.number);
IndexOutput out = directory().createOutput(si.getNormFileName(this.number));
@@ -134,55 +329,39 @@
}
this.dirty = false;
}
-
- /** Closes the underlying IndexInput for this norm.
- * It is still valid to access all other norm properties after close is called.
- * @throws IOException
- */
- private synchronized void close() throws IOException {
- if (in != null && !useSingleNormStream) {
- in.close();
- }
- in = null;
- }
}
-
- /**
- * Increments the RC of this reader, as well as
- * of all norms this reader is using
- */
+
public synchronized void incRef() {
super.incRef();
Iterator it = norms.values().iterator();
while (it.hasNext()) {
- Norm norm = (Norm) it.next();
- norm.incRef();
+ ((Norm) it.next()).incRef();
}
+ if (deletedDocsRef != null) {
+ deletedDocsRef.incRef();
+ }
}
- /**
- * only increments the RC of this reader, not tof
- * he norms. This is important whenever a reopen()
- * creates a new SegmentReader that doesn't share
- * the norms with this one
- */
private synchronized void incRefReaderNotNorms() {
super.incRef();
}
-
+
public synchronized void decRef() throws IOException {
super.decRef();
Iterator it = norms.values().iterator();
while (it.hasNext()) {
- Norm norm = (Norm) it.next();
- norm.decRef();
+ ((Norm) it.next()).decRef();
}
+
+ if (deletedDocsRef != null) {
+ deletedDocsRef.decRef();
+ }
}
private synchronized void decRefReaderNotNorms() throws IOException {
super.decRef();
}
-
+
Map norms = new HashMap();
/** The class which implements SegmentReader. */
@@ -410,6 +589,7 @@
// NOTE: the bitvector is stored using the regular directory, not cfs
if (hasDeletions(si)) {
deletedDocs = new BitVector(directory(), si.getDelFileName());
+ deletedDocsRef = new Ref();
assert si.getDelCount() == deletedDocs.count() :
"delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count();
@@ -423,66 +603,88 @@
assert si.getDelCount() == 0;
}
- protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException {
+ /**
+ * Clones the norm bytes. May be overridden by subclasses. New and experimental.
+ * @param bv Byte array to clone
+ * @return New BitVector
+ */
+ protected byte[] cloneNormBytes(byte[] bytes) {
+ byte[] cloneBytes = new byte[bytes.length];
+ System.arraycopy(bytes, 0, cloneBytes, 0, bytes.length);
+ return cloneBytes;
+ }
+
+ /**
+ * Clones the deleteDocs BitVector. May be overridden by subclasses. New and experimental.
+ * @param bv BitVector to clone
+ * @return New BitVector
+ */
+ protected BitVector cloneDeletedDocs(BitVector bv) {
+ return (BitVector)bv.clone();
+ }
+
+ protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
DirectoryIndexReader newReader;
if (infos.size() == 1) {
SegmentInfo si = infos.info(0);
if (segment.equals(si.name) && si.getUseCompoundFile() == SegmentReader.this.si.getUseCompoundFile()) {
- newReader = reopenSegment(si);
+ newReader = reopenSegment(si, doClone, openReadOnly);
} else {
// segment not referenced anymore, reopen not possible
// or segment format changed
- newReader = SegmentReader.get(readOnly, infos, infos.info(0), false);
+ newReader = SegmentReader.get(openReadOnly, infos, infos.info(0), false);
}
} else {
- if (readOnly)
- return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null);
+ if (openReadOnly)
+ return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null, doClone);
else
- return new MultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null, false);
+ return new MultiSegmentReader(directory, infos, closeDirectory, new SegmentReader[] {this}, null, null, false, doClone);
}
return newReader;
}
- synchronized SegmentReader reopenSegment(SegmentInfo si) throws CorruptIndexException, IOException {
+ synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
boolean deletionsUpToDate = (this.si.hasDeletions() == si.hasDeletions())
&& (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName()));
boolean normsUpToDate = true;
-
boolean[] fieldNormsChanged = new boolean[fieldInfos.size()];
- if (normsUpToDate) {
- for (int i = 0; i < fieldInfos.size(); i++) {
- if (!this.si.getNormFileName(i).equals(si.getNormFileName(i))) {
- normsUpToDate = false;
- fieldNormsChanged[i] = true;
- }
+ final int fieldCount = fieldInfos.size();
+ for (int i = 0; i < fieldCount; i++) {
+ if (!this.si.getNormFileName(i).equals(si.getNormFileName(i))) {
+ normsUpToDate = false;
+ fieldNormsChanged[i] = true;
}
}
- if (normsUpToDate && deletionsUpToDate) {
+ // if we're cloning we need to run through the reopenSegment logic
+ if (normsUpToDate && deletionsUpToDate && !doClone && openReadOnly == readOnly) {
return this;
}
-
// clone reader
SegmentReader clone;
- if (readOnly)
- clone = new ReadOnlySegmentReader();
- else
- clone = new SegmentReader();
+ try {
+ if (openReadOnly)
+ clone = (SegmentReader) READONLY_IMPL.newInstance();
+ else
+ clone = (SegmentReader) IMPL.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("cannot load SegmentReader class: " + e, e);
+ }
boolean success = false;
try {
- clone.readOnly = readOnly;
+ clone.readOnly = openReadOnly;
clone.directory = directory;
clone.si = si;
clone.segment = segment;
clone.readBufferSize = readBufferSize;
clone.cfsReader = cfsReader;
clone.storeCFSReader = storeCFSReader;
-
+
clone.fieldInfos = fieldInfos;
clone.tis = tis;
clone.freqStream = freqStream;
@@ -493,71 +695,52 @@
clone.fieldsReaderOrig = (FieldsReader) fieldsReaderOrig.clone();
}
- if (!deletionsUpToDate) {
- // load deleted docs
- clone.deletedDocs = null;
- clone.loadDeletedDocs();
+ if (deletedDocsRef != null) {
+ deletedDocsRef.incRef();
+ }
+ if (doClone) {
+ clone.deletedDocs = deletedDocs;
+ clone.deletedDocsRef = deletedDocsRef;
} else {
- clone.deletedDocs = this.deletedDocs;
+ if (!deletionsUpToDate) {
+ // load deleted docs
+ clone.deletedDocs = null;
+ clone.deletedDocsRef = null;
+ clone.loadDeletedDocs();
+ } else {
+ clone.deletedDocs = deletedDocs;
+ clone.deletedDocsRef = deletedDocsRef;
+ }
}
-
+
clone.norms = new HashMap();
- if (!normsUpToDate) {
- // load norms
- for (int i = 0; i < fieldNormsChanged.length; i++) {
- // copy unchanged norms to the cloned reader and incRef those norms
- if (!fieldNormsChanged[i]) {
- String curField = fieldInfos.fieldInfo(i).name;
- Norm norm = (Norm) this.norms.get(curField);
- norm.incRef();
- clone.norms.put(curField, norm);
- }
+
+ // Clone norms
+ for (int i = 0; i < fieldNormsChanged.length; i++) {
+
+ // Clone unchanged norms to the cloned reader
+ if (doClone || !fieldNormsChanged[i]) {
+ final String curField = fieldInfos.fieldInfo(i).name;
+ Norm norm = (Norm) this.norms.get(curField);
+ clone.norms.put(curField, norm.clone());
}
-
- clone.openNorms(si.getUseCompoundFile() ? cfsReader : directory(), readBufferSize);
- } else {
- Iterator it = norms.keySet().iterator();
- while (it.hasNext()) {
- String field = (String) it.next();
- Norm norm = (Norm) norms.get(field);
- norm.incRef();
- clone.norms.put(field, norm);
- }
}
-
- if (clone.singleNormStream == null) {
- for (int i = 0; i < fieldInfos.size(); i++) {
- FieldInfo fi = fieldInfos.fieldInfo(i);
- if (fi.isIndexed && !fi.omitNorms) {
- Directory d = si.getUseCompoundFile() ? cfsReader : directory();
- String fileName = si.getNormFileName(fi.number);
- if (si.hasSeparateNorms(fi.number)) {
- continue;
- }
-
- if (fileName.endsWith("." + IndexFileNames.NORMS_EXTENSION)) {
- clone.singleNormStream = d.openInput(fileName, readBufferSize);
- break;
- }
- }
- }
- }
-
+
+ // If we are not cloning, then this will open anew
+ // any norms that have changed:
+ clone.openNorms(si.getUseCompoundFile() ? cfsReader : directory(), readBufferSize);
+
success = true;
} finally {
if (this.referencedSegmentReader != null) {
- // this reader shares resources with another SegmentReader,
- // so we increment the other readers refCount. We don't
- // increment the refCount of the norms because we did
- // that already for the shared norms
+ // This reader shares resources with another SegmentReader,
+ // so we increment the other reader's refCount.
clone.referencedSegmentReader = this.referencedSegmentReader;
- referencedSegmentReader.incRefReaderNotNorms();
} else {
- // this reader wasn't reopened, so we increment this
- // readers refCount
+ // We are the original SegmentReader
clone.referencedSegmentReader = this;
- incRefReaderNotNorms();
}
+ clone.referencedSegmentReader.incRefReaderNotNorms();
if (!success) {
// An exception occured during reopen, we have to decRef the norms
@@ -570,6 +753,7 @@
}
protected void commitChanges() throws IOException {
+
if (deletedDocsDirty) { // re-write deleted
si.advanceDelGen();
@@ -603,35 +787,24 @@
FieldsReader getFieldsReader() {
return (FieldsReader) fieldsReaderLocal.get();
}
-
+
protected void doClose() throws IOException {
boolean hasReferencedReader = (referencedSegmentReader != null);
termVectorsLocal.close();
fieldsReaderLocal.close();
-
+
if (hasReferencedReader) {
referencedSegmentReader.decRefReaderNotNorms();
referencedSegmentReader = null;
}
- deletedDocs = null;
-
- // close the single norms stream
- if (singleNormStream != null) {
- // we can close this stream, even if the norms
- // are shared, because every reader has it's own
- // singleNormStream
- singleNormStream.close();
- singleNormStream = null;
- }
-
if (!hasReferencedReader) {
// close everything, nothing is shared anymore with other readers
if (tis != null) {
tis.close();
}
-
+
if (freqStream != null)
freqStream.close();
if (proxStream != null)
@@ -676,8 +849,21 @@
}
protected void doDelete(int docNum) {
- if (deletedDocs == null)
+ if (deletedDocs == null) {
deletedDocs = new BitVector(maxDoc());
+ deletedDocsRef = new Ref();
+ }
+ // there is more than 1 SegmentReader with a reference to this
+ // deletedDocs BitVector so decRef the current deletedDocsRef,
+ // clone the BitVector, create a new deletedDocsRef
+ if (deletedDocsRef.refCount() > 1) {
+ synchronized (deletedDocsRef) {
+ Ref oldRef = deletedDocsRef;
+ deletedDocs = cloneDeletedDocs(deletedDocs);
+ deletedDocsRef = new Ref();
+ oldRef.decRef();
+ }
+ }
deletedDocsDirty = true;
undeleteAll = false;
if (!deletedDocs.getAndSet(docNum))
@@ -685,9 +871,16 @@
}
protected void doUndeleteAll() {
+ deletedDocsDirty = false;
+ undeleteAll = true;
+ if (deletedDocs != null) {
+ assert deletedDocsRef != null;
+ deletedDocsRef.decRef();
deletedDocs = null;
- deletedDocsDirty = false;
- undeleteAll = true;
+ deletedDocsRef = null;
+ } else {
+ assert deletedDocsRef == null;
+ }
}
List files() throws IOException {
@@ -844,17 +1037,7 @@
protected synchronized byte[] getNorms(String field) throws IOException {
Norm norm = (Norm) norms.get(field);
if (norm == null) return null; // not indexed, or norms not stored
- synchronized(norm) {
- if (norm.bytes == null) { // value not yet read
- byte[] bytes = new byte[maxDoc()];
- norms(field, bytes, 0);
- norm.bytes = bytes; // cache it
- // it's OK to close the underlying IndexInput as we have cached the
- // norms and will never read them again.
- norm.close();
- }
- return norm.bytes;
- }
+ return norm.bytes();
}
// returns fake norms if norms aren't available
@@ -871,10 +1054,8 @@
if (norm == null) // not an indexed field
return;
- norm.dirty = true; // mark it dirty
normsDirty = true;
-
- norms(field)[doc] = value; // set the value
+ norm.copyOnWrite()[doc] = value; // set the value
}
/** Read norms into a pre-allocated array. */
@@ -887,24 +1068,8 @@
System.arraycopy(fakeNorms(), 0, bytes, offset, maxDoc());
return;
}
-
- synchronized(norm) {
- if (norm.bytes != null) { // can copy from cache
- System.arraycopy(norm.bytes, 0, bytes, offset, maxDoc());
- return;
- }
-
- // Read from disk. norm.in may be shared across multiple norms and
- // should only be used in a synchronized context.
- IndexInput normStream;
- if (norm.useSingleNormStream) {
- normStream = singleNormStream;
- } else {
- normStream = norm.in;
- }
- normStream.seek(norm.normSeek);
- normStream.readBytes(bytes, offset, maxDoc());
- }
+
+ norm.bytes(bytes, offset, maxDoc());
}
@@ -932,8 +1097,11 @@
if (singleNormFile) {
normSeek = nextNormSeek;
- if (singleNormStream==null) {
+ if (singleNormStream == null) {
singleNormStream = d.openInput(fileName, readBufferSize);
+ singleNormRef = new Ref();
+ } else {
+ singleNormRef.incRef();
}
// All norms in the .nrm file can share a single IndexInput since
// they are only used in a synchronized context.
@@ -944,7 +1112,7 @@
normInput = d.openInput(fileName);
}
- norms.put(fi.name, new Norm(normInput, singleNormFile, fi.number, normSeek));
+ norms.put(fi.name, new Norm(normInput, fi.number, normSeek));
nextNormSeek += maxDoc; // increment also if some norms are separate
}
}
@@ -964,11 +1132,11 @@
}
return true;
}
-
+
// for testing only
boolean normsClosed(String field) {
- Norm norm = (Norm) norms.get(field);
- return norm.refCount == 0;
+ Norm norm = (Norm) norms.get(field);
+ return norm.refCount == 0;
}
/**
Index: src/java/org/apache/lucene/index/MultiReader.java
===================================================================
--- src/java/org/apache/lucene/index/MultiReader.java (revision 737779)
+++ src/java/org/apache/lucene/index/MultiReader.java (working copy)
@@ -32,7 +32,7 @@
*
* @version $Id$
*/
-public class MultiReader extends IndexReader {
+public class MultiReader extends IndexReader implements Cloneable {
protected IndexReader[] subReaders;
private int[] starts; // 1st docno for each segment
private boolean[] decrefOnClose; // remember which subreaders to decRef on close
@@ -86,7 +86,7 @@
}
starts[subReaders.length] = maxDoc;
}
-
+
/**
* Tries to reopen the subreaders.
*
@@ -106,53 +106,61 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexReader reopen() throws CorruptIndexException, IOException {
+ public synchronized IndexReader reopen() throws CorruptIndexException, IOException {
+ return doReopen(false);
+ }
+
+ /**
+ * Clones the subreaders.
+ * (see {@link IndexReader#clone()}).
+ *
+ *
+ * If subreaders are shared, then the reference count of those
+ * readers is increased to ensure that the subreaders remain open
+ * until the last referring reader is closed.
+ */
+ public synchronized Object clone() {
+ try {
+ return doReopen(true);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * If clone is true then we clone each of the subreaders
+ * @param doClone
+ * @return New IndexReader, or same one (this) if
+ * reopen/clone is not necessary
+ * @throws CorruptIndexException
+ * @throws IOException
+ */
+ protected IndexReader doReopen(boolean doClone) throws CorruptIndexException, IOException {
ensureOpen();
boolean reopened = false;
IndexReader[] newSubReaders = new IndexReader[subReaders.length];
- boolean[] newDecrefOnClose = new boolean[subReaders.length];
boolean success = false;
try {
for (int i = 0; i < subReaders.length; i++) {
- newSubReaders[i] = subReaders[i].reopen();
+ if (doClone)
+ newSubReaders[i] = (IndexReader) subReaders[i].clone();
+ else
+ newSubReaders[i] = subReaders[i].reopen();
// if at least one of the subreaders was updated we remember that
// and return a new MultiReader
if (newSubReaders[i] != subReaders[i]) {
reopened = true;
- // this is a new subreader instance, so on close() we don't
- // decRef but close it
- newDecrefOnClose[i] = false;
}
}
-
- if (reopened) {
- for (int i = 0; i < subReaders.length; i++) {
- if (newSubReaders[i] == subReaders[i]) {
- newSubReaders[i].incRef();
- newDecrefOnClose[i] = true;
- }
- }
-
- MultiReader mr = new MultiReader(newSubReaders);
- mr.decrefOnClose = newDecrefOnClose;
- success = true;
- return mr;
- } else {
- success = true;
- return this;
- }
+ success = true;
} finally {
if (!success && reopened) {
for (int i = 0; i < newSubReaders.length; i++) {
- if (newSubReaders[i] != null) {
+ if (newSubReaders[i] != subReaders[i]) {
try {
- if (newDecrefOnClose[i]) {
- newSubReaders[i].decRef();
- } else {
- newSubReaders[i].close();
- }
+ newSubReaders[i].close();
} catch (IOException ignore) {
// keep going - we want to clean up as much as possible
}
@@ -160,6 +168,23 @@
}
}
}
+
+ if (reopened) {
+ boolean[] newDecrefOnClose = new boolean[subReaders.length];
+ for (int i = 0; i < subReaders.length; i++) {
+ if (newSubReaders[i] == subReaders[i]) {
+ newSubReaders[i].incRef();
+ newDecrefOnClose[i] = true;
+ }
+ }
+ MultiReader mr = new MultiReader(newSubReaders);
+ mr.decrefOnClose = newDecrefOnClose;
+ success = true;
+ return mr;
+ } else {
+ success = true;
+ return this;
+ }
}
public TermFreqVector[] getTermFreqVectors(int n) throws IOException {
Index: src/java/org/apache/lucene/index/IndexReader.java
===================================================================
--- src/java/org/apache/lucene/index/IndexReader.java (revision 737779)
+++ src/java/org/apache/lucene/index/IndexReader.java (working copy)
@@ -67,7 +67,7 @@
@version $Id$
*/
-public abstract class IndexReader {
+public abstract class IndexReader implements Cloneable {
// NOTE: in 3.0 this will change to true
final static boolean READ_ONLY_DEFAULT = false;
@@ -328,12 +328,13 @@
* If the index has not changed since this instance was (re)opened, then this
* call is a NOOP and returns this instance. Otherwise, a new instance is
* returned. The old instance is not closed and remains usable.
- * Note: The re-opened reader instance and the old instance might share
- * the same resources. For this reason no index modification operations
- * (e. g. {@link #deleteDocument(int)}, {@link #setNorm(int, String, byte)})
- * should be performed using one of the readers until the old reader instance
- * is closed. Otherwise, the behavior of the readers is undefined.
*
+ * If the reader is reopened, even though they share
+ * resources internally, it's safe to make changes
+ * (deletions, norms) with the new reader. All shared
+ * mutable state obeys "copy on write" semantics to ensure
+ * the changes are not seen by other readers.
+ *
* You can determine whether a reader was actually reopened by comparing the
* old instance with the instance returned by this method:
*
@@ -354,7 +355,49 @@
public synchronized IndexReader reopen() throws CorruptIndexException, IOException {
throw new UnsupportedOperationException("This reader does not support reopen().");
}
+
+ /** Just like {@link #reopen()}, except you can change the
+ * readOnly of the original reader. If the index is
+ * unchanged but readOnly is different then a new reader
+ * will be returned. */
+ public synchronized IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
+ throw new UnsupportedOperationException("This reader does not support reopen().");
+ }
+
+ /**
+ * Efficiently clones the IndexReader (sharing most
+ * internal state).
+ *
+ * On cloning a reader with pending changes (deletions,
+ * norms), the original reader transfers its write lock to
+ * the cloned reader. This means only the cloned reader
+ * may make further changes to the index, and commit the
+ * changes to the index on close, but the old reader still
+ * reflects all changes made up until it was cloned.
+ *
+ * Like {@link #reopen()}, it's safe to make changes to
+ * either the original or the cloned reader: all shared
+ * mutable state obeys "copy on write" semantics to ensure
+ * the changes are not seen by other readers.
+ *
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ public synchronized Object clone() {
+ throw new UnsupportedOperationException("This reader does not implement clone()");
+ }
+
+ /**
+ * Clones the IndexReader and optionally changes readOnly. A readOnly
+ * reader cannot open a writeable reader.
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ public synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
+ throw new UnsupportedOperationException("This reader does not implement clone()");
+ }
+
/**
* Returns the directory associated with this index. The Default
* implementation returns the directory specified by subclasses when
Index: src/java/org/apache/lucene/index/DocumentsWriter.java
===================================================================
--- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 737779)
+++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy)
@@ -919,9 +919,13 @@
if (reader != null) {
try {
if (success)
- reader.doCommit();
+ reader.commit();
} finally {
- reader.doClose();
+ // Force reader to not have changes; if we hit
+ // an exception during commit, we don't want
+ // close to retry the commit:
+ reader.hasChanges = false;
+ reader.close();
}
}
}
Index: src/java/org/apache/lucene/index/MultiSegmentReader.java
===================================================================
--- src/java/org/apache/lucene/index/MultiSegmentReader.java (revision 737779)
+++ src/java/org/apache/lucene/index/MultiSegmentReader.java (working copy)
@@ -32,7 +32,7 @@
/**
* An IndexReader which reads indexes with multiple segments.
*/
-class MultiSegmentReader extends DirectoryIndexReader {
+class MultiSegmentReader extends DirectoryIndexReader implements Cloneable {
protected SegmentReader[] subReaders;
private int[] starts; // 1st docno for each segment
private Map normsCache = new HashMap();
@@ -70,7 +70,8 @@
}
/** This contructor is only used for {@link #reopen()} */
- MultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache, boolean readOnly) throws IOException {
+ MultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts,
+ Map oldNormsCache, boolean readOnly, boolean doClone) throws IOException {
super(directory, infos, closeDirectory, readOnly);
// we put the old SegmentReaders in a map, that allows us
@@ -108,7 +109,20 @@
// this is a new reader; in case we hit an exception we can close it safely
newReader = SegmentReader.get(readOnly, infos.info(i));
} else {
- newReader = (SegmentReader) newReaders[i].reopenSegment(infos.info(i));
+ newReader = (SegmentReader) newReaders[i].reopenSegment(infos.info(i), doClone, readOnly);
+ if (newReader == newReaders[i] && newReaders[i].hasSegmentInfos()) {
+ // Special case when a single-segment reader was
+ // reopened to a multi-segment reader -- we must
+ // get a private clone, to clear its
+ // SegmentInfos, so it does not attempt to
+ // obtain the write lock
+ newReader = (SegmentReader) newReaders[i].clone(readOnly);
+ newReader.init(directory, null, false, readOnly);
+ }
+
+ // Make sure reopenSegment did not carry over a
+ // segmentInfos instance
+ assert !newReader.hasSegmentInfos();
}
if (newReader == newReaders[i]) {
// this reader will be shared between the old and the new one,
@@ -194,15 +208,15 @@
starts[subReaders.length] = maxDoc;
}
- protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException {
+ protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
if (infos.size() == 1) {
// The index has only one segment now, so we can't refresh the MultiSegmentReader.
// Return a new [ReadOnly]SegmentReader instead
- return SegmentReader.get(readOnly, infos, infos.info(0), false);
- } else if (readOnly) {
- return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache);
+ return SegmentReader.get(openReadOnly, infos, infos.info(0), false);
+ } else if (openReadOnly) {
+ return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, doClone);
} else {
- return new MultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, false);
+ return new MultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, false, doClone);
}
}
Index: src/java/org/apache/lucene/index/ReadOnlyMultiSegmentReader.java
===================================================================
--- src/java/org/apache/lucene/index/ReadOnlyMultiSegmentReader.java (revision 737779)
+++ src/java/org/apache/lucene/index/ReadOnlyMultiSegmentReader.java (working copy)
@@ -27,8 +27,8 @@
super(directory, sis, closeDirectory, true);
}
- ReadOnlyMultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache) throws IOException {
- super(directory, infos, closeDirectory, oldReaders, oldStarts, oldNormsCache, true);
+ ReadOnlyMultiSegmentReader(Directory directory, SegmentInfos infos, boolean closeDirectory, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache, boolean doClone) throws IOException {
+ super(directory, infos, closeDirectory, oldReaders, oldStarts, oldNormsCache, true, doClone);
}
protected void acquireWriteLock() {
Index: src/java/org/apache/lucene/store/SingleInstanceLockFactory.java
===================================================================
--- src/java/org/apache/lucene/store/SingleInstanceLockFactory.java (revision 737779)
+++ src/java/org/apache/lucene/store/SingleInstanceLockFactory.java (working copy)
@@ -80,6 +80,6 @@
}
public String toString() {
- return "SingleInstanceLock: " + lockName;
+ return super.toString() + ": " + lockName;
}
}
Index: src/java/org/apache/lucene/util/BitVector.java
===================================================================
--- src/java/org/apache/lucene/util/BitVector.java (revision 737779)
+++ src/java/org/apache/lucene/util/BitVector.java (working copy)
@@ -35,7 +35,7 @@
@version $Id$
*/
-public final class BitVector {
+public final class BitVector implements Cloneable {
private byte[] bits;
private int size;
@@ -46,7 +46,19 @@
size = n;
bits = new byte[(size >> 3) + 1];
}
-
+
+ BitVector(byte[] bits, int size, int count) {
+ this.bits = bits;
+ this.size = size;
+ this.count = count;
+ }
+
+ public Object clone() {
+ byte[] copyBits = new byte[bits.length];
+ System.arraycopy(bits, 0, copyBits, 0, bits.length);
+ return new BitVector(copyBits, size, count);
+ }
+
/** Sets the value of bit to one. */
public final void set(int bit) {
if (bit >= size) {