diff -Naur lucene-2.3.1/src/java/org/apache/lucene/store/FSDirectory.java lucene-2.3.1_patched/src/java/org/apache/lucene/store/FSDirectory.java --- lucene-2.3.1/src/java/org/apache/lucene/store/FSDirectory.java 2008-02-19 10:58:07.000000000 -0800 +++ lucene-2.3.1_patched/src/java/org/apache/lucene/store/FSDirectory.java 2008-07-02 11:06:36.000000000 -0700 @@ -25,6 +25,9 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Hashtable; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.Queue; +import java.util.concurrent.Semaphore; import org.apache.lucene.index.IndexFileNameFilter; @@ -495,6 +498,77 @@ protected static class FSIndexInput extends BufferedIndexInput { + class DescriptorPool { + private final int MAX_AVAILABLE = 64; + private final Semaphore available = new Semaphore(MAX_AVAILABLE, true); + private final Queue files = new ConcurrentLinkedQueue(); + private File file; + private String mode; + long length; + + public DescriptorPool(File file, String mode) throws IOException { + this.file = file; + this.mode = mode; + + Descriptor newFile = new Descriptor(file, mode); + length = newFile.length(); + files.add(newFile); + } + + public Descriptor get() throws IOException { + available.acquireUninterruptibly(1); + Descriptor nextFile; + try { + nextFile = getOrCreateFile(); + } catch (IOException ie) { + available.release(); + throw(ie); + } + + return nextFile; + } + + public void release(Descriptor file) { + releaseFile(file); + available.release(); + } + + + private Descriptor getOrCreateFile() throws IOException { + Descriptor nextFile = null; + nextFile = (Descriptor)files.poll(); + + if (nextFile == null) { + // the semaphore will prevent more than MAX_AVAILABLE + // descriptors from being open + nextFile = new Descriptor(file,mode); + } + return nextFile; + } + private void releaseFile(Descriptor file) { + files.add(file); + } + + public long length() { + return length; + } + + public void close() throws IOException { + // Aquire all permits then close and throw away all files + available.acquireUninterruptibly(MAX_AVAILABLE); + + try { + while(files.size() > 0) { + Descriptor nextFile = (Descriptor)files.poll(); + nextFile.close(); + } + } finally { + available.release(MAX_AVAILABLE); + } + } + } + + private static class Descriptor extends RandomAccessFile { // remember if the file is open, so that we don't try to close it // more than once @@ -524,7 +598,7 @@ } } - private final Descriptor file; + private final DescriptorPool filePool; boolean isClone; public FSIndexInput(File path) throws IOException { @@ -533,13 +607,19 @@ public FSIndexInput(File path, int bufferSize) throws IOException { super(bufferSize); - file = new Descriptor(path, "r"); + filePool = new DescriptorPool(path, "r"); } + /** IndexInput methods */ protected void readInternal(byte[] b, int offset, int len) throws IOException { - synchronized (file) { + + Descriptor file = filePool.get(); + + + // Access to file is controlled by the filePool + try { long position = getFilePointer(); if (position != file.position) { file.seek(position); @@ -553,19 +633,21 @@ file.position += i; total += i; } while (total < len); + } finally { + filePool.release(file); } } public void close() throws IOException { // only close the file if this is not a clone - if (!isClone) file.close(); + if (!isClone) filePool.close(); } protected void seekInternal(long position) { } public long length() { - return file.length; + return filePool.length(); } public Object clone() { @@ -578,7 +660,16 @@ * file descriptor is valid. */ boolean isFDValid() throws IOException { - return file.getFD().valid(); + Descriptor file = filePool.get(); + + boolean valid; + try { + valid = file.getFD().valid(); + } finally { + filePool.release(file); + } + + return valid; } }