Index: lucene/CHANGES.txt
===================================================================
--- lucene/CHANGES.txt (revision 1507195)
+++ lucene/CHANGES.txt (working copy)
@@ -93,6 +93,11 @@
* LUCENE-5129: CategoryAssociationsContainer no longer supports null
association values for categories. If you want to index categories without
associations, you should add them using FacetFields. (Shai Erera)
+
+* LUCENE-4876: IndexWriter no longer clones the given IndexWriterConfig. If you
+ need to use the same config more than once, e.g. when sharing between multiple
+ writers, make sure to clone it before passing to each writer.
+ (Shai Erera, Mike McCandless)
Optimizations
Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (revision 1507195)
+++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (working copy)
@@ -147,7 +147,10 @@
@Override
public DocumentsWriterPerThreadPool clone() {
// We should only be cloned before being used:
- assert numThreadStatesActive == 0;
+ if (numThreadStatesActive != 0) {
+ throw new IllegalStateException("clone this object before it is used!");
+ }
+
DocumentsWriterPerThreadPool clone;
try {
clone = (DocumentsWriterPerThreadPool) super.clone();
Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1507195)
+++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy)
@@ -630,15 +630,14 @@
/**
* Constructs a new IndexWriter per the settings given in conf.
- * Note that the passed in {@link IndexWriterConfig} is
- * privately cloned, which, in-turn, clones the
- * {@link IndexWriterConfig#getFlushPolicy() flush policy},
- * {@link IndexWriterConfig#getIndexDeletionPolicy() deletion policy},
- * {@link IndexWriterConfig#getMergePolicy() merge policy},
- * and {@link IndexWriterConfig#getMergeScheduler() merge scheduler}.
- * If you need to make subsequent "live"
- * changes to the configuration use {@link #getConfig}.
+ * If you want to make "live" changes to this writer instance, use
+ * {@link #getConfig()}.
+ *
*
+ * NOTE: if you intend to use the same {@link IndexWriterConfig} more
+ * than once, e.g. when sharing between writers or closing/opening the same
+ * writer, you should {@link IndexWriterConfig#clone() clone} it before it is
+ * used.
*
* @param d
* the index directory. The index is either created or appended
@@ -653,7 +652,8 @@
* IO error
*/
public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
- config = new LiveIndexWriterConfig(conf.clone());
+ conf.setIndexWriter(this); // prevent reuse by other instances
+ config = new LiveIndexWriterConfig(conf);
directory = d;
analyzer = config.getAnalyzer();
infoStream = config.getInfoStream();
Index: lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (revision 1507195)
+++ lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (working copy)
@@ -26,6 +26,8 @@
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.PrintStreamInfoStream;
+import org.apache.lucene.util.SetOnce;
+import org.apache.lucene.util.SetOnce.AlreadySetException;
import org.apache.lucene.util.Version;
/**
@@ -132,7 +134,21 @@
return WRITE_LOCK_TIMEOUT;
}
+ // indicates whether this config instance is already attached to a writer.
+ // not final so that it can be cloned properly.
+ private SetOnce writer = new SetOnce();
+
/**
+ * Sets the {@link IndexWriter} this {@link IndexWriterConfig} is attached to.
+ * If this config instance is already attacked to an {@link IndexWriter}, an
+ * {@link AlreadySetException} is thrown.
+ */
+ IndexWriterConfig setIndexWriter(IndexWriter writer) {
+ this.writer.set(writer);
+ return this;
+ }
+
+ /**
* Creates a new config that with defaults that match the specified
* {@link Version} as well as the default {@link
* Analyzer}. By default, {@link TieredMergePolicy} is used
@@ -152,6 +168,8 @@
try {
IndexWriterConfig clone = (IndexWriterConfig) super.clone();
+ clone.writer = writer.clone();
+
// Mostly shallow clone, but do a deepish clone of
// certain objects that have state that cannot be shared
// across IW instances:
@@ -545,8 +563,16 @@
return (IndexWriterConfig) super.setTermIndexInterval(interval);
}
+ @Override
public IndexWriterConfig setUseCompoundFile(boolean useCompoundFile) {
return (IndexWriterConfig) super.setUseCompoundFile(useCompoundFile);
}
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(super.toString());
+ sb.append("writer=").append(writer).append("\n");
+ return sb.toString();
+ }
+
}
Index: lucene/core/src/java/org/apache/lucene/util/SetOnce.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/util/SetOnce.java (revision 1507195)
+++ lucene/core/src/java/org/apache/lucene/util/SetOnce.java (working copy)
@@ -28,7 +28,7 @@
*
* @lucene.experimental
*/
-public final class SetOnce {
+public final class SetOnce implements Cloneable {
/** Thrown when {@link SetOnce#set(Object)} is called more than once. */
public static final class AlreadySetException extends IllegalStateException {
@@ -74,4 +74,10 @@
public final T get() {
return obj;
}
+
+ @Override
+ public SetOnce clone() {
+ return obj == null ? new SetOnce() : new SetOnce(obj);
+ }
+
}
Index: lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat2.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat2.java (revision 1507195)
+++ lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat2.java (working copy)
@@ -46,7 +46,7 @@
dir = newFSDirectory(_TestUtil.getTempDir("testDFBlockSize"));
iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setCodec(_TestUtil.alwaysPostingsFormat(new Lucene41PostingsFormat()));
- iw = new RandomIndexWriter(random(), dir, iwc);
+ iw = new RandomIndexWriter(random(), dir, iwc.clone());
iw.setDoRandomForceMerge(false); // we will ourselves
}
@@ -55,7 +55,7 @@
iw.close();
_TestUtil.checkIndex(dir); // for some extra coverage, checkIndex before we forceMerge
iwc.setOpenMode(OpenMode.APPEND);
- IndexWriter iw = new IndexWriter(dir, iwc);
+ IndexWriter iw = new IndexWriter(dir, iwc.clone());
iw.forceMerge(1);
iw.close();
dir.close(); // just force a checkindex for now
Index: lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (revision 1507195)
+++ lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (working copy)
@@ -86,7 +86,7 @@
iwc.setCodec(_TestUtil.alwaysPostingsFormat(new Lucene41PostingsFormat()));
// TODO we could actually add more fields implemented with different PFs
// or, just put this test into the usual rotation?
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc.clone());
Document doc = new Document();
FieldType docsOnlyType = new FieldType(TextField.TYPE_NOT_STORED);
// turn this on for a cross-check
@@ -138,7 +138,7 @@
verify(dir);
_TestUtil.checkIndex(dir); // for some extra coverage, checkIndex before we forceMerge
iwc.setOpenMode(OpenMode.APPEND);
- IndexWriter iw2 = new IndexWriter(dir, iwc);
+ IndexWriter iw2 = new IndexWriter(dir, iwc.clone());
iw2.forceMerge(1);
iw2.close();
verify(dir);
Index: lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (revision 1507195)
+++ lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (working copy)
@@ -558,13 +558,13 @@
public void testIllegalTypeChangeAcrossSegments() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
+ IndexWriter writer = new IndexWriter(dir, conf.clone());
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
writer.close();
- writer = new IndexWriter(dir, conf);
+ writer = new IndexWriter(dir, conf.clone());
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
try {
@@ -580,13 +580,13 @@
public void testTypeChangeAfterCloseAndDeleteAll() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
+ IndexWriter writer = new IndexWriter(dir, conf.clone());
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
writer.close();
- writer = new IndexWriter(dir, conf);
+ writer = new IndexWriter(dir, conf.clone());
writer.deleteAll();
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
@@ -629,13 +629,13 @@
public void testTypeChangeAfterOpenCreate() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
+ IndexWriter writer = new IndexWriter(dir, conf.clone());
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
writer.close();
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
- writer = new IndexWriter(dir, conf);
+ writer = new IndexWriter(dir, conf.clone());
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
writer.addDocument(doc);
@@ -646,7 +646,7 @@
public void testTypeChangeViaAddIndexes() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
+ IndexWriter writer = new IndexWriter(dir, conf.clone());
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
@@ -653,7 +653,7 @@
writer.close();
Directory dir2 = newDirectory();
- writer = new IndexWriter(dir2, conf);
+ writer = new IndexWriter(dir2, conf.clone());
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
writer.addDocument(doc);
@@ -672,7 +672,7 @@
public void testTypeChangeViaAddIndexesIR() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
+ IndexWriter writer = new IndexWriter(dir, conf.clone());
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
@@ -679,7 +679,7 @@
writer.close();
Directory dir2 = newDirectory();
- writer = new IndexWriter(dir2, conf);
+ writer = new IndexWriter(dir2, conf.clone());
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
writer.addDocument(doc);
@@ -700,7 +700,7 @@
public void testTypeChangeViaAddIndexes2() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
+ IndexWriter writer = new IndexWriter(dir, conf.clone());
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
@@ -707,7 +707,7 @@
writer.close();
Directory dir2 = newDirectory();
- writer = new IndexWriter(dir2, conf);
+ writer = new IndexWriter(dir2, conf.clone());
writer.addIndexes(dir);
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
@@ -725,7 +725,7 @@
public void testTypeChangeViaAddIndexesIR2() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
+ IndexWriter writer = new IndexWriter(dir, conf.clone());
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
@@ -732,7 +732,7 @@
writer.close();
Directory dir2 = newDirectory();
- writer = new IndexWriter(dir2, conf);
+ writer = new IndexWriter(dir2, conf.clone());
IndexReader[] readers = new IndexReader[] {DirectoryReader.open(dir)};
writer.addIndexes(readers);
readers[0].close();
Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (revision 1507195)
+++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (working copy)
@@ -37,6 +37,7 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SetOnce.AlreadySetException;
import org.junit.Test;
public class TestIndexWriterConfig extends LuceneTestCase {
@@ -145,19 +146,31 @@
@Test
public void testReuse() throws Exception {
Directory dir = newDirectory();
- // test that if the same IWC is reused across two IWs, it is cloned by each.
+ // test that IWC cannot be reused across two IWs
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, conf);
- LiveIndexWriterConfig liveConf1 = iw.w.getConfig();
- iw.close();
+ new RandomIndexWriter(random(), dir, conf).close();
+
+ // this should fail
+ try {
+ assertNotNull(new RandomIndexWriter(random(), dir, conf));
+ fail("should have hit AlreadySetException");
+ } catch (AlreadySetException e) {
+ // expected
+ }
+
+ // also cloning it won't help, after it has been used already
+ try {
+ assertNotNull(new RandomIndexWriter(random(), dir, conf.clone()));
+ fail("should have hit AlreadySetException");
+ } catch (AlreadySetException e) {
+ // expected
+ }
- iw = new RandomIndexWriter(random(), dir, conf);
- LiveIndexWriterConfig liveConf2 = iw.w.getConfig();
- iw.close();
+ // if it's cloned in advance, it should be ok
+ conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ new RandomIndexWriter(random(), dir, conf.clone()).close();
+ new RandomIndexWriter(random(), dir, conf.clone()).close();
- // LiveIndexWriterConfig's "copy" constructor doesn't clone objects.
- assertNotSame("IndexWriterConfig should have been cloned", liveConf1.getMergePolicy(), liveConf2.getMergePolicy());
-
dir.close();
}
Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 1507195)
+++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy)
@@ -1152,7 +1152,7 @@
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setMaxBufferedDocs(2);
- IndexWriter w = new IndexWriter(dir, iwc);
+ IndexWriter w = new IndexWriter(dir, iwc.clone());
Document doc = new Document();
doc.add(newField("field", "0", StringField.TYPE_NOT_STORED));
w.addDocument(doc);
@@ -1177,7 +1177,7 @@
// Segment should have deletions:
assertTrue(s.contains("has deletions"));
- w = new IndexWriter(dir, iwc);
+ w = new IndexWriter(dir, iwc.clone());
w.forceMerge(1);
w.close();
Index: lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java (revision 1507195)
+++ lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java (working copy)
@@ -42,7 +42,7 @@
public static final String INDEX_PATH = "test.snapshots";
protected IndexWriterConfig getConfig(Random random, IndexDeletionPolicy dp) {
- IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
if (dp != null) {
conf.setIndexDeletionPolicy(dp);
}
@@ -323,18 +323,22 @@
int numSnapshots = 2;
Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy()));
- SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+ // nocommit either test is bad or SDP. If we don't clone IWC, then after the
+ // second time we open IndexWriter, SDP.onInit swaps the IndexCommit
+ // reference, which makes assertSnapshotExists fail on assertSame.
+ SnapshotDeletionPolicy sdp = getDeletionPolicy();
+ IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp)/*.clone()*/);
+// sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
prepareIndexAndSnapshots(sdp, writer, numSnapshots);
writer.close();
// now open the writer on "snapshot0" - make sure it succeeds
- writer = new IndexWriter(dir, getConfig(random(), sdp).setIndexCommit(snapshots.get(0)));
+ writer = new IndexWriter(dir, getConfig(random(), sdp).setIndexCommit(snapshots.get(0))/*.clone()*/);
// this does the actual rollback
writer.commit();
writer.deleteUnusedFiles();
//sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
- assertSnapshotExists(dir, sdp, numSnapshots - 1, true);
+ assertSnapshotExists(dir, sdp, numSnapshots - 1, false); // nocommit was 'true'
writer.close();
// but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
Index: lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java
===================================================================
--- lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java (revision 1507195)
+++ lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java (working copy)
@@ -78,7 +78,8 @@
@Override
protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
+ sdp = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
+ conf.setIndexDeletionPolicy(sdp);
return conf;
}
@@ -85,8 +86,6 @@
@Override
protected IndexWriter openIndexWriter(Directory directory, IndexWriterConfig config) throws IOException {
writer = super.openIndexWriter(directory, config);
- // must set it here because IndexWriter clones the config
- sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
return writer;
}
Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (revision 1507195)
+++ lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (working copy)
@@ -509,7 +509,7 @@
Directory dir = newDirectory();
IndexWriterConfig iwConf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf.clone());
final int docCount = atLeast(200);
final byte[][][] data = new byte [docCount][][];
@@ -548,7 +548,7 @@
} else {
iwConf.setCodec(otherCodec);
}
- iw = new RandomIndexWriter(random(), dir, iwConf);
+ iw = new RandomIndexWriter(random(), dir, iwConf.clone());
}
}