+ * This class differs from FSDirectory in that for certain IO operations, + * if they fail with an IO exception, a retry logic is applied. + *
+ * The motivation for retry is that, apparently, Windows FS behavior + * differs from other platforms. Windows API allows Windows applications to + * monitor files, in which case the change or creation of a file can cause the + * monitoring application to examine that file. As result, there might be very + * short intervals in which these files cannot be accessed. In the unfortunate case + * that this is a Lucene index file, the result might be an IOEception thrown, most + * probably saying "Access Denied", and an index can become corrupted if this happens in + * the middle of updating the index. + *
+ * The simple retry applied here would occur only in these rare cases when that + * problem occurs. If the problem does not occur, no retry would take place and there + * should be no performance implications. In those cases that no error occured the + * retry is not applied at all. + *
+ * For more info on this problem see discussion for + * Lucene Issue 665. + *
+ * In order to use this class, it should be made the implementation class for FSDirectorey. + * There are two ways to do this: + *
org.apache.lucene.FSDirectory.class
+ * to org.apache.lucene.store.FSWinDirectory.
+ * For instance, when starting a Java application in the cmomand line, the following command line option
+ * would do it:
+ * java -Dorg.apache.lucene.FSDirectory.class=org.apache.lucene.store.FSWinDirectory.
+ * + * This class does not verify that it is invoked in a Windows platform - this is the user + * responsibilty. It would work in other platforms as well, but the retry logic would not be applied, + * so there is not much point in using it in other platforms. + *
+ * There are three (static) configuration setups when this class is in use: + *
+ * There is an additional configuration setup for the lock:
+ * {@link org.apache.lucene.store.SimpleFSWinLockFactory#enableLockReleaseRetry(boolean)}.
+ *
+ * @author Doron Cohen
+ */
+public class FSWinDirectory extends FSDirectory {
+
+ /** Default total retry delay */
+ public static final long DEFAULT_MAX_TOTAL_DELAY = 240;
+ /** Default retry delay interval */
+ public static final long DEFAULT_INTERVAL_DELAY = 30;
+ /** Default mode for delete retry (disabled) */
+ public static final boolean DEFAULT_DELETE_RETRY = false;
+
+ private static long maxTotalDelay = DEFAULT_MAX_TOTAL_DELAY;
+ private static long intervalDelay = DEFAULT_INTERVAL_DELAY;
+ private static boolean enableDeleteRetry = DEFAULT_DELETE_RETRY;
+
+ public FSWinDirectory() {
+ super();
+ }
+
+ /**
+ * Returns the intervalDelay.
+ * @return the intervalDelay.
+ * @see #setIntervalDelay(long).
+ */
+ public static long getIntervalDelay() {
+ return intervalDelay;
+ }
+
+ /**
+ * Sets the interval before next retry.
+ * @param intervalDelay The intervalDelay to set (millis).
+ */
+ public static void setIntervalDelay(long intervalDelay) {
+ FSWinDirectory.intervalDelay = intervalDelay;
+ }
+
+ /**
+ * Returns the maxTotalDelay.
+ * @return Returns the maxTotalDelay.
+ * @see #setMaxTotalDelay(long).
+ */
+ public static long getMaxTotalDelay() {
+ return maxTotalDelay;
+ }
+
+ /**
+ * Sets the maximum allowed total delay due to retry.
+ * @param maxTotalDelay The maxTotalDelay to set (millis).
+ */
+ public static void setMaxTotalDelay(long maxTotalDelay) {
+ FSWinDirectory.maxTotalDelay = maxTotalDelay;
+ }
+
+ /**
+ * Checks if retry is enabled for file delete.
+ * @return true iff retry is enabled for file delete.
+ * @see #enableDeleteRetry(boolean).
+ */
+ public static boolean isEnableDeleteRetry() {
+ return enableDeleteRetry;
+ }
+
+ /**
+ * Enables or disables retry for file delete.
+ * By default retry is disabled for file delete.
+ * This is because in some calls to this method in Lucene, if a file cannot
+ * be deleted, it is simply saved in a "delete later" list.
+ * For those cases, the retry logic might impose an unnecessary delay in program execution.
+ * But there are other usages where this "delete later" logic is not applied,
+ * where it could be worth to retry.
+ * This is dosabled by default.
+ * @param enabled true to enable, false to disable.
+ */
+ public static void enableDeleteRetry(boolean enabled) {
+ enableDeleteRetry = enabled;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#createOutput(java.lang.String)
+ */
+ public IndexOutput createOutput(String name) throws IOException {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ return super.createOutput(name);
+ } catch (IOException e) {
+ error = e;
+ }
+ }
+ return super.createOutput(name);
+ }
+
+ // wait an interval for retry.
+ // return true if another retry is expected
+ static boolean waitAgain(long maxTime, int trialsSoFar, Exception e) {
+ return waitAgain(maxTime, intervalDelay, trialsSoFar, e);
+ }
+
+ // wait an interval for retry.
+ // return true if another retry is expected
+ static boolean waitAgain(long maxTime, long interval, int trialsSoFar, Exception e) {
+ long moreTime = maxTime - System.currentTimeMillis();
+ long delay = Math.min(moreTime, interval);
+ if (trialsSoFar>0 && delay > 0) { // do not wait at all in first trial
+ //if (e!=null) { // debug
+ // System.out.println("\nRetry because of: "+e);
+ // e.printStackTrace(System.out);
+ //}
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException e1) {
+ return false;
+ }
+ }
+ moreTime = maxTime - System.currentTimeMillis();
+ return (moreTime > 5); // wait again if at least 5 ms left
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#deleteFile(java.lang.String)
+ */
+ public void deleteFile(String name) throws IOException {
+ if (isEnableDeleteRetry()) {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ super.deleteFile(name);
+ return;
+ } catch (IOException e) {
+ error = e;
+ }
+ }
+ }
+ super.deleteFile(name);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#createDefaultFSLockFactory(java.io.File)
+ */
+ protected LockFactory createDefaultFSLockFactory(File lockDir) throws IOException {
+ return new SimpleFSWinLockFactory(lockDir);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#fileExists(java.lang.String)
+ */
+ public boolean fileExists(String name) {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ return super.fileExists(name);
+ } catch (Exception e) { // cannot insist on IOException here
+ error = e;
+ }
+ }
+ return super.fileExists(name);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#fileLength(java.lang.String)
+ */
+ public long fileLength(String name) {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ return super.fileLength(name);
+ } catch (Exception e) { // cannot insist on IOException here
+ error = e;
+ }
+ }
+ return super.fileLength(name);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#fileModified(java.io.File)
+ */
+ protected long fileModified(File file) {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ return super.fileModified(file);
+ } catch (Exception e) { // cannot insist on IOException here
+ error = e;
+ }
+ }
+ return super.fileModified(file);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#list()
+ */
+ public String[] list() {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ return super.list();
+ } catch (Exception e) { // cannot insist on IOException here
+ error = e;
+ }
+ }
+ return super.list();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#openInput(java.lang.String)
+ */
+ public IndexInput openInput(String name) throws IOException {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ return super.openInput(name);
+ } catch (IOException e) {
+ error = e;
+ }
+ }
+ return super.openInput(name);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#renameFile(java.lang.String, java.lang.String)
+ */
+ public synchronized void renameFile(String from, String to) throws IOException {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ super.renameFile(from, to);
+ return;
+ } catch (IOException e) {
+ error = e;
+ }
+ }
+ super.renameFile(from, to);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.store.FSDirectory#touchFile(java.lang.String)
+ */
+ public void touchFile(String name) {
+ int trialsSoFar = 0;
+ long maxTime = System.currentTimeMillis() + maxTotalDelay;
+ Exception error = null;
+ while (waitAgain(maxTime, trialsSoFar++, error)) {
+ try {
+ super.touchFile(name);
+ return;
+ } catch (Exception e) { // cannot insist on IOException here
+ error = e;
+ }
+ }
+ super.touchFile(name);
+ }
+
+}
Index: src/java/org/apache/lucene/store/SimpleFSLockFactory.java
===================================================================
--- src/java/org/apache/lucene/store/SimpleFSLockFactory.java (revision 447480)
+++ src/java/org/apache/lucene/store/SimpleFSLockFactory.java (working copy)
@@ -41,7 +41,7 @@
System.getProperty("org.apache.lucene.lockDir",
System.getProperty("java.io.tmpdir"));
- private File lockDir;
+ protected File lockDir;
/**
* Instantiate using default LOCK_DIR: org.apache.lucene.lockDir
Index: src/java/org/apache/lucene/store/SimpleFSWinLockFactory.java
===================================================================
--- src/java/org/apache/lucene/store/SimpleFSWinLockFactory.java (revision 0)
+++ src/java/org/apache/lucene/store/SimpleFSWinLockFactory.java (revision 0)
@@ -0,0 +1,192 @@
+package org.apache.lucene.store;
+
+/*
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Windows specific implementation of SimpleFSLockFactory.
+ *
+ * This class differs from {@link org.apache.lucene.store.SimpleFSLockFactory} in that + * for certain IO operations, if they fail, a retry logic is applied. + *
+ * See {@link org.apache.lucene.store.FSWinDirectory} for the motivation for retry. + * + * @author Doron Cohen + */ +public class SimpleFSWinLockFactory extends SimpleFSLockFactory { + + public static final boolean DEFAULT_DELETE_RETRY = false; + + private static boolean enableLockReleaseRetry = DEFAULT_DELETE_RETRY; + + /** + * Checks if retry is enabled for lock file delete. + * @return true iff retry is enabled for lock file delete. + * @see #enableLockReleaseRetry(boolean). + */ + public static boolean isEnableLockReleaseRetry() { + return enableLockReleaseRetry; + } + + /** + * Enables or disables retry for lock release. + * By default retry is disabled for lock file release. + * This is because a possible risk in NFS mounted file systems. + * In NFS systems, a file delete attempt might return failure code due to NFS timeout, + * although actually the delete succeeded. There is usually no harm in trying to + * delete again a file that was already deleted. But here this would be dangerous, because + * the file deletion stands for releasing a "co-operative lock file". + * Therefore the recommendation is to enable this feature only in a non-NFS env. + * This is disabled by default. + * @param enabled true to enable, false to disable. + */ + public static void enableLockReleaseRetry(boolean enabled) { + enableLockReleaseRetry = enabled; + } + + /** + * @throws IOException + */ + public SimpleFSWinLockFactory() throws IOException { + super(); + } + + /** + * @param lockDir + * @throws IOException + */ + public SimpleFSWinLockFactory(File lockDir) throws IOException { + super(lockDir); + } + + /** + * @param lockDirName + * @throws IOException + */ + public SimpleFSWinLockFactory(String lockDirName) throws IOException { + super(lockDirName); + } + + /* (non-Javadoc) + * @see org.apache.lucene.store.SimpleFSLockFactory#clearAllLocks() + */ + public void clearAllLocks() throws IOException { + if (enableLockReleaseRetry) { + int trialsSoFar = 0; + long maxTime = System.currentTimeMillis() + FSWinDirectory.getMaxTotalDelay(); + Exception error = null; + while (FSWinDirectory.waitAgain(maxTime, trialsSoFar++, error)) { + try { + super.clearAllLocks(); + return; + } catch (IOException e) { + error = e; + } + } + } + super.clearAllLocks(); + } + + /* (non-Javadoc) + * @see org.apache.lucene.store.SimpleFSLockFactory#makeLock(java.lang.String) + */ + public Lock makeLock(String lockName) { + return new SimpleWinFSLock(lockDir, lockPrefix + "-" + lockName); + } + +} + +class SimpleWinFSLock extends SimpleFSLock { + + /* (non-Javadoc) + * @see org.apache.lucene.store.SimpleFSLock#isLocked() + */ + public boolean isLocked() { + int trialsSoFar = 0; + long maxTime = System.currentTimeMillis() + FSWinDirectory.getMaxTotalDelay(); + Exception error = null; + while (FSWinDirectory.waitAgain(maxTime, trialsSoFar++, error)) { + try { + return super.isLocked(); + } catch (Exception e) { // cannot insist on IOException here + error = e; + } + } + return super.isLocked(); + } + + /* (non-Javadoc) + * @see org.apache.lucene.store.Lock#obtain(long) + */ + public boolean obtain(long lockWaitTimeout) throws IOException { + int trialsSoFar = 0; + long maxTime = System.currentTimeMillis() + lockWaitTimeout; + long delay = LOCK_POLL_INTERVAL; + Exception error = null; + while (FSWinDirectory.waitAgain(maxTime, delay, trialsSoFar++, error)) { + try { + if (obtain()) { + return true; + } + delay = LOCK_POLL_INTERVAL; + } catch (IOException e) { + delay = FSWinDirectory.getIntervalDelay(); + error = e; + } + } + // one trial left + try { + if (obtain()) { + return true; + } + } catch (IOException e) { + //failed + IOException e2 = new IOException("Lock obtain timed out: " + this.toString()); + e2.initCause(e); + throw e2; + } + //failed + throw new IOException("Lock obtain timed out: " + this.toString()); + } + + + /* (non-Javadoc) + * @see org.apache.lucene.store.SimpleFSLock#release() + */ + public void release() { + if (SimpleFSWinLockFactory.isEnableLockReleaseRetry()) { + int trialsSoFar = 0; + long maxTime = System.currentTimeMillis() + FSWinDirectory.getMaxTotalDelay(); + Exception error = null; + while (FSWinDirectory.waitAgain(maxTime, trialsSoFar++, error)) { + try { + super.release(); + return; + } catch (Exception e) { // cannot insist on IOException here + error = e; + } + } + } + super.release(); + } + + public SimpleWinFSLock(File lockDir, String lockFileName) { + super(lockDir, lockFileName); + } +} Index: src/test/org/apache/lucene/store/TestLockFactory.java =================================================================== --- src/test/org/apache/lucene/store/TestLockFactory.java (revision 447480) +++ src/test/org/apache/lucene/store/TestLockFactory.java (working copy) @@ -130,7 +130,7 @@ IndexWriter writer = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), true); assertTrue("FSDirectory did not use correct LockFactory: got " + writer.getDirectory().getLockFactory(), - SimpleFSLockFactory.class.isInstance(writer.getDirectory().getLockFactory())); + isSimpleFsLockFactory(writer.getDirectory().getLockFactory())); IndexWriter writer2 = null; @@ -157,7 +157,7 @@ IndexWriter writer = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), true); assertTrue("FSDirectory did not use correct LockFactory: got " + writer.getDirectory().getLockFactory(), - SimpleFSLockFactory.class.isInstance(writer.getDirectory().getLockFactory())); + isSimpleFsLockFactory(writer.getDirectory().getLockFactory())); // Create a 2nd IndexWriter. This should not fail: IndexWriter writer2 = null; @@ -175,6 +175,10 @@ rmDir(indexDirName); } + //verify that the correct simple fs lock factory is in use + protected boolean isSimpleFsLockFactory(LockFactory lockFactory) { + return SimpleFSLockFactory.class.isInstance(lockFactory); + } // Verify: setting custom lock factory class (as system property) works: // Verify: FSDirectory does basic locking correctly Index: src/test/org/apache/lucene/store/TestWinLockfactory.java =================================================================== --- src/test/org/apache/lucene/store/TestWinLockfactory.java (revision 0) +++ src/test/org/apache/lucene/store/TestWinLockfactory.java (revision 0) @@ -0,0 +1,40 @@ +package org.apache.lucene.store; + +public class TestWinLockfactory extends TestLockFactory { + + String origFsDirImplName = null; + + public TestWinLockfactory() { + super(); + } + + /* (non-Javadoc) + * @see junit.framework.TestCase#setUp() + */ + protected void setUp() throws Exception { + super.setUp(); + origFsDirImplName = FSDirectory.getFSDirImplClass(); + FSDirectory.setFSDirImplClass("org.apache.lucene.store.FSWinDirectory"); + } + + /* (non-Javadoc) + * @see junit.framework.TestCase#tearDown() + */ + protected void tearDown() throws Exception { + try { + super.tearDown(); + } finally { + FSDirectory.setFSDirImplClass(origFsDirImplName); + } + } + + /* (non-Javadoc) + * @see org.apache.lucene.store.TestLockFactory#isSimpleFsLockFactory(org.apache.lucene.store.LockFactory) + */ + protected boolean isSimpleFsLockFactory(LockFactory lockFactory) { + return SimpleFSWinLockFactory.class.isInstance(lockFactory); + } + + + +}