Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 437916) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -50,6 +50,12 @@ */ private static final Hashtable DIRECTORIES = new Hashtable(); + /** + * millis to wait before retrying an IO operation that failed due to perhaps + * a temporary instability of the File System or the Operating System. + */ + static final int UNEXPECTED_ERROR_RETRY_DELAY = 100; + private static boolean disableLocks = false; // TODO: should this move up to the Directory base class? Also: should we @@ -283,6 +289,15 @@ } private synchronized void create() throws IOException { + try { + doCreate(); + } catch (IOException e) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, e, "create"); + doCreate(); + } + } + + private void doCreate() throws IOException { if (!directory.exists()) if (!directory.mkdirs()) throw new IOException("Cannot create directory: " + directory); @@ -304,49 +319,96 @@ /** Returns an array of strings, one for each Lucene index file in the directory. */ public String[] list() { - return directory.list(new IndexFileNameFilter()); + try { + return directory.list(new IndexFileNameFilter()); + } catch (Throwable t) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, t, "list"); + return directory.list(new IndexFileNameFilter()); + } } /** Returns true iff a file with the given name exists. */ public boolean fileExists(String name) { File file = new File(directory, name); - return file.exists(); + try { + return file.exists(); + } catch (Throwable t) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, t, "fileExists"); + return file.exists(); + } } /** Returns the time the named file was last modified. */ public long fileModified(String name) { File file = new File(directory, name); - return file.lastModified(); + try { + return file.lastModified(); + } catch (Throwable t) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, t, "fileModified"); + return file.lastModified(); + } } /** Returns the time the named file was last modified. */ public static long fileModified(File directory, String name) { File file = new File(directory, name); - return file.lastModified(); + try { + return file.lastModified(); + } catch (Throwable t) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, t, "fileModified"); + return file.lastModified(); + } } /** Set the modified time of an existing file to now. */ public void touchFile(String name) { File file = new File(directory, name); - file.setLastModified(System.currentTimeMillis()); + try { + file.setLastModified(System.currentTimeMillis()); + } catch (Throwable t) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, t, "touchFile"); + file.setLastModified(System.currentTimeMillis()); + } } /** Returns the length in bytes of a file in the directory. */ public long fileLength(String name) { File file = new File(directory, name); - return file.length(); + try { + return file.length(); + } catch (Throwable t) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, t, "fileLength"); + return file.length(); + } } /** Removes an existing file in the directory. */ public void deleteFile(String name) throws IOException { File file = new File(directory, name); - if (!file.delete()) + boolean deleted = false; + try { + deleted = file.delete(); + } catch (Throwable t) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, t, "deleteFile"); + deleted = file.delete(); + } + if (!deleted) { throw new IOException("Cannot delete " + file); + } } /** Renames an existing file in the directory. */ public synchronized void renameFile(String from, String to) - throws IOException { + throws IOException { + try { + doRenameFile(from, to); + } catch (IOException e) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, e, "renameFile"); + doRenameFile(from, to); + } + } + + private void doRenameFile(String from, String to) throws IOException { File old = new File(directory, from); File nu = new File(directory, to); @@ -415,14 +477,38 @@ if (file.exists() && !file.delete()) // delete existing, if any throw new IOException("Cannot overwrite: " + file); - return new FSIndexOutput(file); + try { + return new FSIndexOutput(file); + } catch (IOException e) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, e, "createOutput"); + return new FSIndexOutput(file); + } } /** Returns a stream reading an existing file. */ public IndexInput openInput(String name) throws IOException { - return new FSIndexInput(new File(directory, name)); + try { + return new FSIndexInput(new File(directory, name)); + } catch (IOException e) { + waitForRetryIoOp(UNEXPECTED_ERROR_RETRY_DELAY, e, "openInput"); + return new FSIndexInput(new File(directory, name)); + } } + /** wait a short time, before retrying an io operation, to bypass temporary access issues. + * @return true iff waiting was completed (not interrupted). + * @throws IOException */ + static boolean waitForRetryIoOp(long delay, Throwable t, String msg) { + //System.out.println("wait "+delay+" millis and retry an IO operation ("+msg+")"); + //if (t!=null) t.printStackTrace(System.out); + try { + Thread.sleep(delay); + return true; + } catch (InterruptedException ie) { + return false; + } + } + /** * So we can do some byte-to-hexchar conversion below */ Index: src/java/org/apache/lucene/store/SimpleFSLockFactory.java =================================================================== --- src/java/org/apache/lucene/store/SimpleFSLockFactory.java (revision 437916) +++ src/java/org/apache/lucene/store/SimpleFSLockFactory.java (working copy) @@ -80,6 +80,15 @@ } public void clearAllLocks() throws IOException { + try { + doClearAllLocks(); + } catch (IOException e) { + FSDirectory.waitForRetryIoOp(FSDirectory.UNEXPECTED_ERROR_RETRY_DELAY, e, "clearAllLocks"); + doClearAllLocks(); + } + } + + private void doClearAllLocks() throws IOException { String[] files = lockDir.list(); if (files == null) throw new IOException("Cannot read lock directory " + @@ -119,12 +128,55 @@ return lockFile.createNewFile(); } + // overide here - apply retry logic also for unexpected IO exceptions. + public boolean obtain(long lockWaitTimeout) throws IOException { + long tmax = System.currentTimeMillis() + lockWaitTimeout; + IOException unexpectedError = null; + while (true) { + try { + unexpectedError = null; + if (obtain()) { + return true; + } + } catch (IOException e){ + unexpectedError = e; + } + + long timeLeft = tmax - System.currentTimeMillis(); + if (timeLeft <= 0) { + break; + } + + long interval = (unexpectedError!=null ? FSDirectory.UNEXPECTED_ERROR_RETRY_DELAY : LOCK_POLL_INTERVAL); + interval = Math.min(interval, timeLeft); + if(!FSDirectory.waitForRetryIoOp(interval, unexpectedError, "lock.obtain(timeout)")) { + break; + } + } + + IOException ex = new IOException("Lock obtain timed out: " + this.toString()); + if (unexpectedError != null) { + ex.initCause(unexpectedError); + } + throw ex; + } + public void release() { - lockFile.delete(); + try { + lockFile.delete(); + } catch (Throwable t) { + FSDirectory.waitForRetryIoOp(FSDirectory.UNEXPECTED_ERROR_RETRY_DELAY, t, "lock.release"); + lockFile.delete(); + } } public boolean isLocked() { - return lockFile.exists(); + try { + return lockFile.exists(); + } catch (Throwable t) { + FSDirectory.waitForRetryIoOp(FSDirectory.UNEXPECTED_ERROR_RETRY_DELAY, t, "lock.isLocked"); + return lockFile.exists(); + } } public String toString() {