Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 948865) +++ lucene/CHANGES.txt (working copy) @@ -61,6 +61,10 @@ CREATE*, IndexWriter would make a first empty commit. If you need that behavior you can call writer.commit()/close() immediately after you create it. (Shai Erera, Mike McCandless) + +* LUCENE-2481: SnapshotDeletionPolicy.snapshot() and release() were replaced + with equivalent ones that take a String (id) as argument. You can pass + whatever ID you want, as long as you use it when calling both. (Shai Erera) Changes in runtime behavior @@ -330,6 +334,10 @@ * LUCENE-2440: Add support for custom ExecutorService in ParallelMultiSearcher (Edward Drapkin via Mike McCandless) +* LUCENE-2481: SnapshotDeletionPolicy supports multiple snapshots, that can be + identified by an ID (String). Also PersistentSnapshotDeletionPolicy can be + used to persist the snapshots information on to stable storage. (Shai Erera) + Optimizations * LUCENE-2075: Terms dict cache is now shared across threads instead Index: lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java =================================================================== --- lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (revision 0) @@ -0,0 +1,180 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import java.io.IOException; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.Version; + +/** + * A {@link SnapshotDeletionPolicy} which adds a persistence layer so that + * snapshots can be maintained across the life of an application. The snapshots + * are persisted in a {@link Directory} and are committed as soon as + * {@link #snapshot(String)} or {@link #release(String)} is called. + *

+ * NOTE: this class receives a {@link Directory} to persist the data into + * a Lucene index. It is highly recommended to use a dedicated directory (and on + * stable storage as well) for persisting the snapshots' information, and not + * reuse the content index directory, or otherwise conflicts and index + * corruptions will occur. + *

+ * NOTE: you should call {@link #close()} when you're done using this + * class for safetyness (it will close the {@link IndexWriter} instance used). + */ +public class PersistentSnapshotDeletionPolicy extends SnapshotDeletionPolicy { + + // Used to validate that the given directory includes just one document w/ the + // given ID field. Otherwise, it's not a valid Directory for snapshotting. + private static final String SNAPSHOTS_ID = "$SNAPSHOTS_DOC$"; + + // The index writer which maintains the snapshots metadata + private final IndexWriter writer; + + /** + * {@link PersistentSnapshotDeletionPolicy} wraps another + * {@link IndexDeletionPolicy} to enable flexible snapshotting. + * + * @param primary + * the {@link IndexDeletionPolicy} that is used on non-snapshotted + * commits. Snapshotted commits, by definition, are not deleted until + * explicitly released via {@link #release(String)}. + * @param dir + * the {@link Directory} which will be used to persist the snapshots + * information. + * @param mode + * specifies whether a new index should be created, deleting all + * existing snapshots information (immediately), or open an existing + * index, initializing the class with the snapsthots information. + * @param matchVersion + * specifies the {@link Version} that should be used when opening the + * IndexWriter. + */ + public PersistentSnapshotDeletionPolicy(IndexDeletionPolicy primary, + Directory dir, OpenMode mode, Version matchVersion) throws CorruptIndexException, + LockObtainFailedException, IOException { + super(primary, null); + + // Initialize the index writer over the snapshot directory. + writer = new IndexWriter(dir, new IndexWriterConfig(matchVersion, null).setOpenMode(mode)); + if (mode != OpenMode.APPEND) { + // IndexWriter no longer creates a first commit on an empty Directory. So + // if we were asked to CREATE*, call commit() just to be sure. If the + // index contains information and mode is CREATE_OR_APPEND, it's a no-op. + writer.commit(); + } + + // Initializes the snapshots information. This code should basically run + // only if mode != CREATE, but if it is, it's no harm as we only open the + // reader once and immediately close it. + IndexReader r = writer.getReader(); + try { + int numDocs = r.numDocs(); + // index is allowed to have exactly one document or 0. + if (numDocs == 1) { + Document doc = r.document(r.maxDoc() - 1); + Field sid = doc.getField(SNAPSHOTS_ID); + if (sid == null) { + throw new IllegalStateException("directory is not a valid snapshots store!"); + } + for (Fieldable f : doc.getFields()) { + if (f.stringValue().length() > 0) { // don't add the SNAPSHOTS_ID field + registerSnapshotInfo(f.name(), f.stringValue(), null); + } + } + } else if (numDocs != 0) { + throw new IllegalStateException( + "should be only 1 document in the snapshots directory: " + numDocs); + } + } finally { + r.close(); + } + } + + @Override + public synchronized void onInit(List commits) + throws IOException { + // super.onInit() needs to be called first to ensure that initialization + // behaves as expected. The superclass, SnapshotDeletionPolicy, ensures + // that any snapshot IDs with empty IndexCommits are released. Since this + // happens, this class needs to persist these changes. + super.onInit(commits); + persistSnapshotInfos(null, null); + } + + /** + * Snapshots the last commit using the given ID. Once this method returns, the + * snapshot information is persisted in the directory. + * + * @see SnapshotDeletionPolicy#snapshot(String) + */ + @Override + public synchronized IndexCommit snapshot(String id) throws IOException { + checkSnapshotted(id); + if (SNAPSHOTS_ID.equals(id)) { + throw new IllegalArgumentException(id + " is reserved and cannot be used as a snapshot id"); + } + persistSnapshotInfos(id, lastCommit.getSegmentsFileName()); + return super.snapshot(id); + } + + /** + * Deletes a snapshotted commit by ID. Once this method returns, the snapshot + * information is committed to the directory. + * + * @see SnapshotDeletionPolicy#release(String) + */ + @Override + public synchronized void release(String id) throws IOException { + super.release(id); + persistSnapshotInfos(null, null); + } + + /** Closes the index which writes the snapshots to the directory. */ + public void close() throws CorruptIndexException, IOException { + writer.close(); + } + + /** + * Persists all snapshots information. If the given id and segment are not + * null, it persists their information as well. + */ + private void persistSnapshotInfos(String id, String segment) throws IOException { + writer.deleteAll(); + Document d = new Document(); + d.add(new Field(SNAPSHOTS_ID, "", Store.YES, Index.NO)); + for (Entry e : super.getSnapshots().entrySet()) { + d.add(new Field(e.getKey(), e.getValue(), Store.YES, Index.NO)); + } + if (id != null) { + d.add(new Field(id, segment, Store.YES, Index.NO)); + } + writer.addDocument(d); + writer.commit(); + } + +} Property changes on: lucene\src\java\org\apache\lucene\index\PersistentSnapshotDeletionPolicy.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java (revision 948865) +++ lucene/src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java (working copy) @@ -18,131 +18,340 @@ */ import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.ArrayList; import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; import java.io.IOException; + import org.apache.lucene.store.Directory; -/** A {@link IndexDeletionPolicy} that wraps around any other - * {@link IndexDeletionPolicy} and adds the ability to hold and - * later release a single "snapshot" of an index. While - * the snapshot is held, the {@link IndexWriter} will not - * remove any files associated with it even if the index is - * otherwise being actively, arbitrarily changed. Because - * we wrap another arbitrary {@link IndexDeletionPolicy}, this - * gives you the freedom to continue using whatever {@link - * IndexDeletionPolicy} you would normally want to use with your - * index. Note that you can re-use a single instance of - * SnapshotDeletionPolicy across multiple writers as long - * as they are against the same index Directory. Any - * snapshot held when a writer is closed will "survive" - * when the next writer is opened. - * - * @lucene.experimental */ - +/** + * A {@link IndexDeletionPolicy} that wraps around any other + * {@link IndexDeletionPolicy} and adds the ability to hold and later release + * snapshots of an index. While a snapshot is held, the {@link IndexWriter} will + * not remove any files associated with it even if the index is otherwise being + * actively, arbitrarily changed. Because we wrap another arbitrary + * {@link IndexDeletionPolicy}, this gives you the freedom to continue using + * whatever {@link IndexDeletionPolicy} you would normally want to use with your + * index. + * + *

+ * This class maintains all snapshots in-memory, and so the information is not + * persisted and not protected against system failures. If persistency is + * important, one can persist the snapshots on to stable storage and when + * creating a new instance of this deletion policy, pass the persistent + * snapshots to {@link #SnapshotDeletionPolicy(IndexDeletionPolicy, Map)}. + * + * @lucene.experimental + */ public class SnapshotDeletionPolicy implements IndexDeletionPolicy { - private IndexCommit lastCommit; + /** Snapshots info */ + private Map idToSnapshot = new HashMap(); + + // multiple IDs could point to the same commit point (segment name) + private Map> segmentToIDs = new HashMap>(); + + /** Holds a Snapshot's information. */ + private static class SnapshotInfo { + String id; + String segment; + IndexCommit commit; + + public SnapshotInfo(String id, String segment, IndexCommit commit) { + this.id = id; + this.segment = segment; + this.commit = commit; + } + + @Override + public String toString() { + return id + " : " + segment; + } + } + + protected IndexCommit lastCommit; private IndexDeletionPolicy primary; - private String snapshot; public SnapshotDeletionPolicy(IndexDeletionPolicy primary) { this.primary = primary; } - public synchronized void onInit(List commits) throws IOException { + /** + * {@link SnapshotDeletionPolicy} wraps another {@link IndexDeletionPolicy} to + * enable flexible snapshotting. + * + * @param primary + * the {@link IndexDeletionPolicy} that is used on non-snapshotted + * commits. Snapshotted commits, are not deleted until explicitly + * released via {@link #release(String)} + * @param snapshotsInfo + * A mapping of snapshot ID to the segment filename that is being + * snapshotted. The expected input would be the output of + * {@link #getSnapshots()}. A null value signals that there are no + * initial snapshots to maintain. + */ + public SnapshotDeletionPolicy(IndexDeletionPolicy primary, + Map snapshotsInfo) { + this(primary); + + if (snapshotsInfo != null) { + // Add the ID->segmentIDs here - the actual IndexCommits will be + // reconciled on the call to onInit() + for (Entry e : snapshotsInfo.entrySet()) { + registerSnapshotInfo(e.getKey(), e.getValue(), null); + } + } + } + + /** Registers the given snapshot information. */ + protected void registerSnapshotInfo(String id, String segment, + IndexCommit commit) { + idToSnapshot.put(id, new SnapshotInfo(id, segment, commit)); + Set ids = segmentToIDs.get(segment); + if (ids == null) { + ids = new HashSet(); + segmentToIDs.put(segment, ids); + } + ids.add(id); + } + + protected void checkSnapshotted(String id) { + if (isSnapshotted(id)) { + throw new IllegalStateException("Snapshot ID " + id + + " is already used - must be unique"); + } + } + + public synchronized void onInit(List commits) + throws IOException { primary.onInit(wrapCommits(commits)); - lastCommit = commits.get(commits.size()-1); + lastCommit = commits.get(commits.size() - 1); + + /* + * Assign snapshotted IndexCommits to their correct snapshot IDs as + * specified in the constructor. + */ + for (IndexCommit commit : commits) { + Set ids = segmentToIDs.get(commit.getSegmentsFileName()); + if (ids != null) { + for (String id : ids) { + idToSnapshot.get(id).commit = commit; + } + } + } + + /* + * Second, see if there are any instances where a snapshot ID was specified + * in the constructor but an IndexCommit doesn't exist. In this case, the ID + * should be removed. + * + * Note: This code is protective for extreme cases where IDs point to + * non-existent segments. As the constructor should have received its + * information via a call to getSnapshots(), the data should be well-formed. + */ + // Find lost snapshots + ArrayList idsToRemove = null; + for (Entry e : idToSnapshot.entrySet()) { + if (e.getValue().commit == null) { + if (idsToRemove == null) { + idsToRemove = new ArrayList(); + } + idsToRemove.add(e.getKey()); + } + } + // Finally, remove those 'lost' snapshots. + if (idsToRemove != null) { + for (String id : idsToRemove) { + SnapshotInfo info = idToSnapshot.remove(id); + segmentToIDs.remove(info.segment); + } + } } - public synchronized void onCommit(List commits) throws IOException { + public synchronized void onCommit(List commits) + throws IOException { primary.onCommit(wrapCommits(commits)); - lastCommit = commits.get(commits.size()-1); + lastCommit = commits.get(commits.size() - 1); } - /** Take a snapshot of the most recent commit to the - * index. You must call release() to free this snapshot. - * Note that while the snapshot is held, the files it - * references will not be deleted, which will consume - * additional disk space in your index. If you take a - * snapshot at a particularly bad time (say just before - * you call optimize()) then in the worst case this could - * consume an extra 1X of your total index size, until - * you release the snapshot. */ - public synchronized IndexCommit snapshot() { + /** + * Snapshots the last commit. Once a commit is 'snapshotted,' it is protected + * from deletion (as long as this {@link IndexDeletionPolicy} is used). The + * commit can be removed by calling {@link #release(String)} using the same ID + * parameter followed by a call to {@link IndexWriter#deleteUnusedFiles()}. + *

+ * NOTE: ID must be unique in the system. If the same ID is used twice, + * an [@link {@link IllegalStateException} is thrown. + *

+ * NOTE: while the snapshot is held, the files it references will not + * be deleted, which will consume additional disk space in your index. If you + * take a snapshot at a particularly bad time (say just before you call + * optimize()) then in the worst case this could consume an extra 1X of your + * total index size, until you release the snapshot. + * + * @param id + * a unique identifier of the commit that is being snapshotted. + * @throws IllegalStateException + * if either there is no 'last commit' to snapshot, or if the + * parameter 'ID' refers to an already snapshotted commit. + * @return the {@link IndexCommit} that was snapshotted. + */ + public synchronized IndexCommit snapshot(String id) throws IOException { if (lastCommit == null) { - throw new IllegalStateException("no index commits to snapshot !"); + // no commit exists. Really shouldn't happen, but might be if SDP is + // accessed before onInit or onCommit was called. + throw new IllegalStateException("No index commits to snapshot"); } - - if (snapshot == null) - snapshot = lastCommit.getSegmentsFileName(); - else - throw new IllegalStateException("snapshot is already set; please call release() first"); + + // Can't use the same snapshot ID twice... + checkSnapshotted(id); + + registerSnapshotInfo(id, lastCommit.getSegmentsFileName(), lastCommit); return lastCommit; } - /** Release the currently held snapshot. */ - public synchronized void release() { - if (snapshot != null) - snapshot = null; - else - throw new IllegalStateException("snapshot was not set; please call snapshot() first"); + /** + * Returns true if the given ID is already used by a snapshot. You can call + * this method before {@link #snapshot(String)} if you are not sure whether + * the ID is already used or not. + */ + public boolean isSnapshotted(String id) { + return idToSnapshot.containsKey(id); } - private class MyCommitPoint extends IndexCommit { + /** + * Release a snapshotted commit by ID. + * + * @param id + * a unique identifier of the commit that is un-snapshotted. + * @throws IllegalStateException + * if no snapshot exists by this ID. + */ + public synchronized void release(String id) throws IOException { + SnapshotInfo info = idToSnapshot.remove(id); + if (info == null) { + throw new IllegalStateException("Snapshot doesn't exist: " + id); + } + Set ids = segmentToIDs.get(info.segment); + if (ids != null) { + ids.remove(id); + if (ids.size() == 0) { + segmentToIDs.remove(info.segment); + } + } + } + + /** + * Get a snapshotted IndexCommit by ID. The IndexCommit can then be used to + * open an IndexReader on a specific commit point, or even rollback to a + * certain commit by opening a new IndexWriter with the IndexCommit specified + * in its {@link IndexWriterConfig}. + * + * @param id + * a unique identifier of the commit that was snapshotted. + * @throws IllegalStateException + * if no snapshot exists by the specified ID. + * @return The IndexCommit point for this particular snapshot. + */ + public synchronized IndexCommit getSnapshot(String id) { + SnapshotInfo snapshotInfo = idToSnapshot.get(id); + if (snapshotInfo == null) { + throw new IllegalStateException("No snapshot exists by ID: " + id); + } + return snapshotInfo.commit; + } + + /** + * Get all the snapshots in a map of snapshot IDs to the segments they + * 'cover.' This can be used in another {@link SnapshotDeletionPolicy} + * constructor to initialize snapshots at construction. + */ + public synchronized Map getSnapshots() { + Map snapshots = new HashMap(); + + for (Entry e : idToSnapshot.entrySet()) { + snapshots.put(e.getKey(), e.getValue().segment); + } + return snapshots; + } + + protected class SnapshotCommitPoint extends IndexCommit { IndexCommit cp; - MyCommitPoint(IndexCommit cp) { + + protected SnapshotCommitPoint(IndexCommit cp) { this.cp = cp; } + @Override public String getSegmentsFileName() { return cp.getSegmentsFileName(); } + @Override public Collection getFileNames() throws IOException { return cp.getFileNames(); } + @Override public Directory getDirectory() { return cp.getDirectory(); } + @Override public void delete() { - synchronized(SnapshotDeletionPolicy.this) { + synchronized (SnapshotDeletionPolicy.this) { // Suppress the delete request if this commit point is // our current snapshot. - if (snapshot == null || !snapshot.equals(getSegmentsFileName())) + if (shouldDelete(getSegmentsFileName())) cp.delete(); } } + + /** + * Returns true if this segment can be deleted. The default implementation + * returns false if this segment is currently held as snapshot. + */ + protected boolean shouldDelete(String segmentsFileName) { + return !segmentToIDs.containsKey(segmentsFileName); + } + @Override public boolean isDeleted() { return cp.isDeleted(); } + @Override public long getVersion() { return cp.getVersion(); } + @Override public long getGeneration() { return cp.getGeneration(); } + @Override - public Map getUserData() throws IOException { + public Map getUserData() throws IOException { return cp.getUserData(); } + @Override public boolean isOptimized() { return cp.isOptimized(); } } - private List wrapCommits(List commits) { - final int count = commits.size(); - List myCommits = new ArrayList(count); - for(int i=0;i wrapCommits(List commits) { + List wrappedCommits = new ArrayList(commits.size()); + for (IndexCommit ic : commits) { + wrappedCommits.add(new SnapshotCommitPoint(ic)); + } + return wrappedCommits; } + } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 948865) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -4769,7 +4769,7 @@ assertEquals(1, IndexReader.listCommits(dir).size()); // Keep that commit - sdp.snapshot(); + sdp.snapshot("id"); // Second commit - now KeepOnlyLastCommit cannot delete the prev commit. doc = new Document(); @@ -4779,7 +4779,7 @@ assertEquals(2, IndexReader.listCommits(dir).size()); // Should delete the unreferenced commit - sdp.release(); + sdp.release("id"); writer.deleteUnusedFiles(); assertEquals(1, IndexReader.listCommits(dir).size()); Index: lucene/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java (revision 0) +++ lucene/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java (revision 0) @@ -0,0 +1,138 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockRAMDirectory; +import org.junit.Test; + +public class TestPersistentSnapshotDeletionPolicy extends TestSnapshotDeletionPolicy { + + // Keep it a class member so that getDeletionPolicy can use it + private Directory snapshotDir = new MockRAMDirectory(); + + @Override + protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException { + IndexWriter.unlock(snapshotDir); + return new PersistentSnapshotDeletionPolicy( + new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.CREATE, + TEST_VERSION_CURRENT); + } + + @Override + protected SnapshotDeletionPolicy getDeletionPolicy(Map snapshots) throws IOException { + SnapshotDeletionPolicy sdp = getDeletionPolicy(); + if (snapshots != null) { + for (Entry e: snapshots.entrySet()) { + sdp.registerSnapshotInfo(e.getKey(), e.getValue(), null); + } + } + return sdp; + } + + @Override + @Test + public void testExistingSnapshots() throws Exception { + int numSnapshots = 3; + Directory dir = new MockRAMDirectory(); + PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) getDeletionPolicy(); + IndexWriter writer = new IndexWriter(dir, getConfig(psdp)); + prepareIndexAndSnapshots(psdp, writer, numSnapshots, "snapshot"); + writer.close(); + psdp.close(); + + // Re-initialize and verify snapshots were persisted + psdp = new PersistentSnapshotDeletionPolicy( + new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND, + TEST_VERSION_CURRENT); + new IndexWriter(dir, getConfig(psdp)).close(); + + assertSnapshotExists(dir, psdp, numSnapshots); + assertEquals(numSnapshots, psdp.getSnapshots().size()); + psdp.close(); + } + + @Test(expected=IllegalArgumentException.class) + public void testIllegalSnapshotId() throws Exception { + getDeletionPolicy().snapshot("$SNAPSHOTS_DOC$"); + } + + @Test + public void testInvalidSnapshotInfos() throws Exception { + // Add the correct number of documents (1), but without snapshot information + IndexWriter writer = new IndexWriter(snapshotDir, getConfig(null)); + writer.addDocument(new Document()); + writer.close(); + try { + new PersistentSnapshotDeletionPolicy( + new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND, + TEST_VERSION_CURRENT); + fail("should not have succeeded to read from an invalid Directory"); + } catch (IllegalStateException e) { + } + } + + @Test + public void testNoSnapshotInfos() throws Exception { + // Initialize an empty index in snapshotDir - PSDP should initialize successfully. + new IndexWriter(snapshotDir, getConfig(null)).close(); + new PersistentSnapshotDeletionPolicy( + new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND, + TEST_VERSION_CURRENT).close(); + } + + @Test(expected=IllegalStateException.class) + public void testTooManySnapshotInfos() throws Exception { + // Write two documents to the snapshots directory - illegal. + IndexWriter writer = new IndexWriter(snapshotDir, getConfig(null)); + writer.addDocument(new Document()); + writer.addDocument(new Document()); + writer.close(); + + new PersistentSnapshotDeletionPolicy( + new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND, + TEST_VERSION_CURRENT).close(); + fail("should not have succeeded to open an invalid directory"); + } + + @Test + public void testSnapshotRelease() throws Exception { + Directory dir = new MockRAMDirectory(); + PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) getDeletionPolicy(); + IndexWriter writer = new IndexWriter(dir, getConfig(psdp)); + prepareIndexAndSnapshots(psdp, writer, 1, "snapshot"); + writer.close(); + + psdp.release("snapshot0"); + psdp.close(); + + psdp = new PersistentSnapshotDeletionPolicy( + new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND, + TEST_VERSION_CURRENT); + assertEquals("Should have no snapshots !", 0, psdp.getSnapshots().size()); + } + +} Property changes on: lucene\src\test\org\apache\lucene\index\TestPersistentSnapshotDeletionPolicy.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java (revision 948865) +++ lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java (working copy) @@ -1,6 +1,9 @@ package org.apache.lucene.index; +import static org.junit.Assert.*; + import java.util.Collection; +import java.util.Map; import java.io.File; import java.io.IOException; @@ -10,25 +13,77 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.MockRAMDirectory; +import org.apache.lucene.analysis.KeywordAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.SnapshotDeletionPolicy; +import org.apache.lucene.util.LuceneTestCaseJ4; import org.apache.lucene.util.ThreadInterruptedException; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.junit.Test; // // This was developed for Lucene In Action, // http://lucenebook.com // -public class TestSnapshotDeletionPolicy extends LuceneTestCase { +public class TestSnapshotDeletionPolicy extends LuceneTestCaseJ4 { public static final String INDEX_PATH = "test.snapshots"; + protected IndexWriterConfig getConfig(IndexDeletionPolicy dp) { + IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer()); + if (dp != null) { + conf.setIndexDeletionPolicy(dp); + } + return conf; + } + + protected void checkSnapshotExists(Directory dir, IndexCommit c) throws Exception { + String segFileName = c.getSegmentsFileName(); + assertTrue("segments file not found in directory: " + segFileName, dir.fileExists(segFileName)); + } + + protected void checkMaxDoc(IndexCommit commit, int expectedMaxDoc) throws Exception { + IndexReader reader = IndexReader.open(commit, true); + try { + assertEquals(expectedMaxDoc, reader.maxDoc()); + } finally { + reader.close(); + } + } + + protected void prepareIndexAndSnapshots(SnapshotDeletionPolicy vdp, + IndexWriter writer, int numSnapshots, String snapshotPrefix) + throws RuntimeException, IOException { + for (int i = 0; i < numSnapshots; i++) { + // create dummy document to trigger commit. + writer.addDocument(new Document()); + writer.commit(); + vdp.snapshot(snapshotPrefix + i); + } + } + + protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException { + return getDeletionPolicy(null); + } + + protected SnapshotDeletionPolicy getDeletionPolicy(Map snapshots) throws IOException { + return new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), snapshots); + } + + protected void assertSnapshotExists(Directory dir, SnapshotDeletionPolicy sdp, int numSnapshots) throws Exception { + for (int i = 0; i < numSnapshots; i++) { + IndexCommit snapshot = sdp.getSnapshot("snapshot" + i); + checkMaxDoc(snapshot, i + 1); + checkSnapshotExists(dir, snapshot); + } + } + + @Test public void testSnapshotDeletionPolicy() throws Exception { File dir = _TestUtil.getTempDir(INDEX_PATH); try { @@ -44,57 +99,11 @@ dir2.close(); } - public void testReuseAcrossWriters() throws Exception { - Directory dir = new MockRAMDirectory(); - - SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, - new StandardAnalyzer(TEST_VERSION_CURRENT)).setIndexDeletionPolicy(dp) - .setMaxBufferedDocs(2)); - Document doc = new Document(); - doc.add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); - for(int i=0;i<7;i++) { - writer.addDocument(doc); - if (i % 2 == 0) { - writer.commit(); - } - } - IndexCommit cp = dp.snapshot(); - copyFiles(dir, cp); - writer.close(); - copyFiles(dir, cp); - - writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, - new StandardAnalyzer(TEST_VERSION_CURRENT)).setIndexDeletionPolicy(dp)); - copyFiles(dir, cp); - for(int i=0;i<7;i++) { - writer.addDocument(doc); - if (i % 2 == 0) { - writer.commit(); - } - } - copyFiles(dir, cp); - writer.close(); - copyFiles(dir, cp); - dp.release(); - writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, - new StandardAnalyzer(TEST_VERSION_CURRENT)).setIndexDeletionPolicy(dp)); - writer.close(); - try { - copyFiles(dir, cp); - fail("did not hit expected IOException"); - } catch (IOException ioe) { - // expected - } - dir.close(); - } - private void runTest(Directory dir) throws Exception { // Run for ~1 seconds final long stopTime = System.currentTimeMillis() + 1000; - SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); + SnapshotDeletionPolicy dp = getDeletionPolicy(); final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new StandardAnalyzer(TEST_VERSION_CURRENT)).setIndexDeletionPolicy(dp) @@ -155,20 +164,21 @@ TestIndexWriter.assertNoUnreferencedFiles(dir, "some files were not deleted but should have been"); } - /** Example showing how to use the SnapshotDeletionPolicy - * to take a backup. This method does not really do a - * backup; instead, it reads every byte of every file - * just to test that the files indeed exist and are - * readable even while the index is changing. */ + /** + * Example showing how to use the SnapshotDeletionPolicy to take a backup. + * This method does not really do a backup; instead, it reads every byte of + * every file just to test that the files indeed exist and are readable even + * while the index is changing. + */ public void backupIndex(Directory dir, SnapshotDeletionPolicy dp) throws Exception { // To backup an index we first take a snapshot: try { - copyFiles(dir, dp.snapshot()); + copyFiles(dir, dp.snapshot("id")); } finally { // Make sure to release the snapshot, otherwise these // files will never be deleted during this IndexWriter // session: - dp.release(); + dp.release("id"); } } @@ -215,5 +225,210 @@ input.close(); } } + + + @Test + public void testBasicSnapshots() throws Exception { + int numSnapshots = 3; + SnapshotDeletionPolicy sdp = getDeletionPolicy(); + + // Create 3 snapshots: snapshot0, snapshot1, snapshot2 + Directory dir = new MockRAMDirectory(); + IndexWriter writer = new IndexWriter(dir, getConfig(sdp)); + prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot"); + writer.close(); + + assertSnapshotExists(dir, sdp, numSnapshots); + + // open a reader on a snapshot - should succeed. + IndexReader.open(sdp.getSnapshot("snapshot0"), true).close(); + + // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone. + sdp = getDeletionPolicy(); + writer = new IndexWriter(dir, getConfig(sdp)); + writer.deleteUnusedFiles(); + writer.close(); + assertEquals("no snapshots should exist", 1, IndexReader.listCommits(dir).size()); + + for (int i = 0; i < numSnapshots; i++) { + try { + sdp.getSnapshot("snapshot" + i); + fail("snapshot shouldn't have existed, but did: snapshot" + i); + } catch (IllegalStateException e) { + // expected - snapshot should not exist + } + } + } + + @Test + public void testMultiThreadedSnapshotting() throws Exception { + Directory dir = new MockRAMDirectory(); + final SnapshotDeletionPolicy sdp = getDeletionPolicy(); + final IndexWriter writer = new IndexWriter(dir, getConfig(sdp)); + + Thread[] threads = new Thread[10]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread() { + @Override + public void run() { + try { + writer.addDocument(new Document()); + writer.commit(); + sdp.snapshot(getName()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + threads[i].setName("t" + i); + } + + for (Thread t : threads) { + t.start(); + } + + for (Thread t : threads) { + t.join(); + } + + // Do one last commit, so that after we release all snapshots, we stay w/ one commit + writer.addDocument(new Document()); + writer.commit(); + + for (Thread t : threads) { + sdp.release(t.getName()); + writer.deleteUnusedFiles(); + } + assertEquals(1, IndexReader.listCommits(dir).size()); + writer.close(); + } + + @Test + public void testRollbackToOldSnapshot() throws Exception { + int numSnapshots = 2; + Directory dir = new MockRAMDirectory(); + SnapshotDeletionPolicy sdp = getDeletionPolicy(); + IndexWriter writer = new IndexWriter(dir, getConfig(sdp)); + prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot"); + writer.close(); + + // now open the writer on "snapshot0" - make sure it succeeds + writer = new IndexWriter(dir, getConfig(sdp).setIndexCommit(sdp.getSnapshot("snapshot0"))); + // this does the actual rollback + writer.commit(); + writer.deleteUnusedFiles(); + assertSnapshotExists(dir, sdp, numSnapshots - 1); + + // but 'snapshot1' files will still exist (need to release snapshot before they can be deleted). + String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName(); + assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName)); + } + + @Test + public void testReleaseSnapshot() throws Exception { + Directory dir = new MockRAMDirectory(); + SnapshotDeletionPolicy sdp = getDeletionPolicy(); + IndexWriter writer = new IndexWriter(dir, getConfig(sdp)); + prepareIndexAndSnapshots(sdp, writer, 1, "snapshot"); + + // Create another commit - we must do that, because otherwise the "snapshot" + // files will still remain in the index, since it's the last commit. + writer.addDocument(new Document()); + writer.commit(); + + // Release + String snapId = "snapshot0"; + String segFileName = sdp.getSnapshot(snapId).getSegmentsFileName(); + sdp.release(snapId); + try { + sdp.getSnapshot(snapId); + fail("should not have succeeded to get an unsnapshotted id"); + } catch (IllegalStateException e) { + // expected + } + assertNull(sdp.getSnapshots().get(snapId)); + writer.deleteUnusedFiles(); + assertFalse("segments file should not be found in dirctory: " + segFileName, dir.fileExists(segFileName)); + } + + @Test + public void testExistingSnapshots() throws Exception { + // Tests the ability to construct a SDP from existing snapshots, and + // asserts that those snapshots/commit points are protected. + int numSnapshots = 3; + Directory dir = new MockRAMDirectory(); + SnapshotDeletionPolicy sdp = getDeletionPolicy(); + IndexWriter writer = new IndexWriter(dir, getConfig(sdp)); + prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot"); + writer.close(); + + // Make a new policy and initialize with snapshots. + sdp = getDeletionPolicy(sdp.getSnapshots()); + writer = new IndexWriter(dir, getConfig(sdp)); + // attempt to delete unused files - the snapshotted files should not be deleted + writer.deleteUnusedFiles(); + writer.close(); + assertSnapshotExists(dir, sdp, numSnapshots); + } + + @Test + public void testSnapshotLastCommitTwice() throws Exception { + Directory dir = new MockRAMDirectory(); + SnapshotDeletionPolicy sdp = getDeletionPolicy(); + IndexWriter writer = new IndexWriter(dir, getConfig(sdp)); + writer.addDocument(new Document()); + writer.commit(); + + String s1 = "s1"; + String s2 = "s2"; + IndexCommit ic1 = sdp.snapshot(s1); + IndexCommit ic2 = sdp.snapshot(s2); + assertTrue(ic1 == ic2); // should be the same instance + + // create another commit + writer.addDocument(new Document()); + writer.commit(); + + // release "s1" should not delete "s2" + sdp.release(s1); + writer.deleteUnusedFiles(); + checkSnapshotExists(dir, ic2); + + writer.close(); + } + + @Test + public void testMissingCommits() throws Exception { + // Tests the behavior of SDP when commits that are given at ctor are missing + // on onInit(). + Directory dir = new MockRAMDirectory(); + SnapshotDeletionPolicy sdp = getDeletionPolicy(); + IndexWriter writer = new IndexWriter(dir, getConfig(sdp)); + writer.addDocument(new Document()); + writer.commit(); + IndexCommit ic = sdp.snapshot("s1"); + + // create another commit, not snapshotted. + writer.addDocument(new Document()); + writer.close(); + + // open a new writer w/ KeepOnlyLastCommit policy, so it will delete "s1" + // commit. + new IndexWriter(dir, getConfig(null)).close(); + + assertFalse("snapshotted commit should not exist", dir.fileExists(ic.getSegmentsFileName())); + + // Now reinit SDP from the commits in the index - the snapshot id should not + // exist anymore. + sdp = getDeletionPolicy(sdp.getSnapshots()); + new IndexWriter(dir, getConfig(sdp)).close(); + + try { + sdp.getSnapshot("s1"); + fail("snapshot s1 should not exist"); + } catch (IllegalStateException e) { + // expected. + } + } + } -