Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 688305) +++ CHANGES.txt (working copy) @@ -114,6 +114,14 @@ hashCode() and equals() in Token, and fixed all core and contrib analyzers to use the re-use APIs. (DM Smith via Mike McCandless) +18. LUCENE-753: Added new Directory implementation + org.apache.lucene.store.NIOFSDirectory, which uses java nio's + FileChannel to do file reads. On most non-Windows platforms, with + many threads sharing a single searcher, this may yield sizable + query throughput when compared to FSDirectory which only allows a + single thread to read from an open file at a time. (Jason + Rutherglen via Mike McCandless) + Bug fixes 1. LUCENE-1134: Fixed BooleanQuery.rewrite to only optimize a single Index: src/java/org/apache/lucene/store/NIOFSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 0) +++ src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 0) @@ -0,0 +1,234 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +/** + * NIO version of FSDirectory. Uses FileChannel.read(ByteBuffer dst, long position) method + * which allows multiple threads to read from the file without synchronizing. FSDirectory + * synchronizes in the FSIndexInput.readInternal method which can cause slowness when there + * are many threads accessing the Directory concurrently. + * + * Note: NIOFSDirectory is not recommended on Windows because of a bug + * in how FileChannel.read is implemented. Inside of the implementation + * the position is synchronized. + * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734 + * + * @see FSDirectory + */ +public class NIOFSDirectory extends FSDirectory { + + // Inherit javadoc + public IndexInput openInput(String name, int bufferSize) throws IOException { + ensureOpen(); + return new NIOFSIndexInput(new File(getFile(), name), bufferSize); + } + + protected static class NIOFSIndexInput extends IndexInput { + + private static class Descriptor extends RandomAccessFile { + // remember if the file is open, so that we don't try to close it + // more than once + private volatile boolean isOpen; + long position; + final long length; + final FileChannel channel; + + public Descriptor(File file, String mode) throws IOException { + super(file, mode); + isOpen = true; + length = length(); + channel = this.getChannel(); + } + + public void close() throws IOException { + if (isOpen) { + isOpen = false; + super.close(); + } + } + + protected void finalize() throws Throwable { + try { + close(); + } finally { + super.finalize(); + } + } + } + + private final Descriptor file; + boolean isClone; + + static final int BUFFER_SIZE = BufferedIndexInput.BUFFER_SIZE; + + private byte[] buffer; + private ByteBuffer bytebuf; // wraps the buffer for NIO + private long bufferStart = 0; // position in file of buffer + private int bufferLength = 0; // end of valid bytes + private int bufferPosition = 0; // next byte to read + + public NIOFSIndexInput(File path) throws IOException { + this(path, BufferedIndexInput.BUFFER_SIZE); + } + + public NIOFSIndexInput(File path, int bufferSize) throws IOException { + file = new Descriptor(path, "r"); + } + + public void readBytes(byte[] b, int offset, int len) throws IOException { + if (len <= (bufferLength - bufferPosition)) { + // the buffer contains enough data to satistfy this request + if (len > 0) // to allow b to be null if len is 0... + System.arraycopy(buffer, bufferPosition, b, offset, len); + bufferPosition += len; + } else { + // the buffer does not have enough data. First serve all we've got. + int available = bufferLength - bufferPosition; + if (available > 0) { + System.arraycopy(buffer, bufferPosition, b, offset, available); + offset += available; + len -= available; + bufferPosition += available; + } + + // and now, read the remaining 'len' bytes: + if (len < BUFFER_SIZE) { + // If the amount left to read is small enough, do it in the usual + // buffered way: fill the buffer and copy from it: + refill(); + if (bufferLength < len) { + // Throw an exception when refill() could not read len bytes: + System.arraycopy(buffer, 0, b, offset, bufferLength); + throw new IOException("read past EOF"); + + } else { + System.arraycopy(buffer, 0, b, offset, len); + bufferPosition = len; + } + } else { + // The amount left to read is larger than the buffer - there's no + // performance reason not to read it all at once. Note that unlike + // the previous code of this function, there is no need to do a seek + // here, because there's no need to reread what we had in the buffer. + long after = bufferStart + bufferPosition + len; + readInternal(b, offset, len); + bufferStart = after; + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read + } + } + } + + private void read(ByteBuffer bb) throws IOException { + long pos = getFilePointer(); + while (bb.hasRemaining()) { + int i = file.channel.read(bb, pos); + if (i == -1) + throw new IOException("read past EOF"); + pos += i; + } + } + + private void refill() throws IOException { + long start = bufferStart + bufferPosition; + long end = start + BUFFER_SIZE; + if (end > length()) + end = length(); + int newLength = (int) (end - start); + if (newLength <= 0) + throw new IOException("read past EOF"); + + if (buffer == null) { + buffer = new byte[BUFFER_SIZE]; // allocate buffer lazily + bytebuf = ByteBuffer.wrap(buffer); + } else { + bytebuf.clear(); + } + + // since we try to read a complete buffer, we need to limit it if + // it's past the end of the file. + if (newLength != BUFFER_SIZE) + bytebuf.limit(newLength); + + read(bytebuf); + + bufferLength = newLength; + bufferStart = start; + bufferPosition = 0; + } + + private void readInternal(byte[] b, int offset, int len) throws IOException { + read(ByteBuffer.wrap(b, offset, len)); + } + + public byte readByte() throws IOException { + if (bufferPosition >= bufferLength) + refill(); + return buffer[bufferPosition++]; + } + + public long getFilePointer() { + return bufferStart + bufferPosition; + } + + public long length() { + return file.length; + } + + public void seek(long pos) throws IOException { + if (pos >= bufferStart && pos < (bufferStart + bufferLength)) + bufferPosition = (int) (pos - bufferStart); // seek within buffer + else { + bufferStart = pos; + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read() + } + } + + public Object clone() { + NIOFSIndexInput clone = (NIOFSIndexInput) super.clone(); + clone.isClone = true; + if (buffer != null) { + clone.buffer = new byte[buffer.length]; + System.arraycopy(buffer, 0, clone.buffer, 0, bufferLength); + clone.bytebuf = ByteBuffer.wrap(clone.buffer); + } + return clone; + } + + public void close() throws IOException { + // only close the file if this is not a clone + if (!isClone) + file.close(); + } + + /** + * Method used for testing. Returns true if the underlying file descriptor + * is valid. + */ + boolean isFDValid() throws IOException { + return file.getFD().valid(); + } + } +} Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 688305) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -544,7 +544,7 @@ private static class Descriptor extends RandomAccessFile { // remember if the file is open, so that we don't try to close it // more than once - private boolean isOpen; + private volatile boolean isOpen; long position; final long length; @@ -633,7 +633,7 @@ // remember if the file is open, so that we don't try to close it // more than once - private boolean isOpen; + private volatile boolean isOpen; public FSIndexOutput(File path) throws IOException { file = new RandomAccessFile(path, "rw");