Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 995019) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -68,9 +68,11 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util._TestUtil; @@ -5134,8 +5136,7 @@ } public void testRandomStoredFields() throws IOException { - File index = _TestUtil.getTempDir("lucenerandfields"); - Directory dir = FSDirectory.open(index); + Directory dir = newDirectory(random); Random rand = random; RandomIndexWriter w = new RandomIndexWriter(rand, dir, newIndexWriterConfig(rand, TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(_TestUtil.nextInt(rand, 5, 20))); //w.w.setInfoStream(System.out); @@ -5221,7 +5222,6 @@ } w.close(); dir.close(); - _TestUtil.rmDir(index); } private static class FailTwiceDuringMerge extends MockDirectoryWrapper.Failure { Index: lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 995019) +++ lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (working copy) @@ -24,6 +24,8 @@ import java.nio.channels.FileChannel; import java.util.concurrent.Future; // javadoc +import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput; + /** * An {@link FSDirectory} implementation that uses java.nio's FileChannel's * positional read, which allows multiple threads to read from the same file @@ -78,6 +80,61 @@ return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize()); } + + @Override + public IndexOutput createOutput(String name) throws IOException { + ensureOpen(); + ensureCanWrite(name); + return new NIOFSIndexOutput(this, name); + } + + /** same as FSDirectory's but with an optimized copyBytes */ + protected static class NIOFSIndexOutput extends FSDirectory.FSIndexOutput { + public NIOFSIndexOutput(FSDirectory parent, String name) throws IOException { + super(parent, name); + } + + @Override + public void copyBytes(DataInput input, long numBytes) throws IOException { + // Optimized copy only if the number of bytes to copy is larger than the + // buffer size, and the given IndexInput supports FileChannel copying. + // NOTE: the below check relies on NIOIndexInput extending Simple. If that + // changes in the future, we should change the check as well. + if (numBytes <= BUFFER_SIZE || !(input instanceof SimpleFSIndexInput)) { + super.copyBytes(input, numBytes); + return; + } + + SimpleFSIndexInput fsInput = (SimpleFSIndexInput) input; + + // flush any bytes in the input's buffer. + numBytes -= fsInput.flushBuffer(this, numBytes); + + // flush any bytes in the buffer + flush(); + + // do the optimized copy + FileChannel in = fsInput.file.getChannel(); + + // Necessary because BufferedIndexInput does lazy seeking: + in.position(fsInput.getFilePointer()); + + FileChannel out = file.getChannel(); + long pos = out.position(); + long writeTo = numBytes + pos; + while (pos < writeTo) { + pos += out.transferFrom(in, pos, Math.min(CHANNEL_CHUNK_SIZE, writeTo - pos)); + } + // transferFrom does not change the position of the channel. Need to change it manually + out.position(pos); + + // corrects the position in super (BufferedIndexOutput), so that calls + // to getFilePointer will return the correct pointer. + // Perhaps a specific method is better? + super.seek(out.position()); + } + } + protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput { private ByteBuffer byteBuf; // wraps the buffer for NIO Index: lucene/src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/FSDirectory.java (revision 995019) +++ lucene/src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -140,7 +140,7 @@ * Chunk size used to read when using FileChannel API. If an attempt to read a * large file is made without limiting the chunk size, an OOM may occur. */ - private static final long CHANNEL_CHUNK_SIZE = 1 << 21; // Use 2MB chunk size - LUCENE-2537 + protected static final long CHANNEL_CHUNK_SIZE = 1 << 21; // Use 2MB chunk size - LUCENE-2537 // returns the canonical version of the directory, creating it if it doesn't exist. private static File getCanonicalPath(File file) throws IOException { @@ -450,10 +450,29 @@ return chunkSize; } + /** + * Copies the content of a given {@link FileChannel} to a destination one. The + * copy is done in chunks of 2MB because if transferFrom is used without a + * limit when copying a very large file, then an OOM may be thrown (depends on + * the state of the RAM in the machine, as well as the OS used). Performance + * measurements showed that chunk sizes larger than 2MB do not result in much + * faster file copy, therefore we limit the size to be safe with different + * file sizes and systems. + */ + static void copy(FileChannel input, FileChannel output, long numBytes) throws IOException { + long pos = output.position(); + long writeTo = numBytes + pos; + while (pos < writeTo) { + pos += output.transferFrom(input, pos, Math.min(CHANNEL_CHUNK_SIZE, writeTo - pos)); + } + // transferFrom does not change the position of the channel. Need to change it manually + output.position(pos); + } + protected static class FSIndexOutput extends BufferedIndexOutput { private final FSDirectory parent; private final String name; - private final RandomAccessFile file; + protected final RandomAccessFile file; private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once public FSIndexOutput(FSDirectory parent, String name) throws IOException { @@ -472,41 +491,23 @@ @Override public void copyBytes(DataInput input, long numBytes) throws IOException { // Optimized copy only if the number of bytes to copy is larger than the - // buffer size, and the given IndexInput supports FileChannel copying. + // buffer size, and the given IndexInput supports FileChannel copying .. // NOTE: the below check relies on NIOIndexInput extending Simple. If that // changes in the future, we should change the check as well. - if (numBytes <= BUFFER_SIZE || !(input instanceof SimpleFSIndexInput)) { + if (numBytes > BUFFER_SIZE && input instanceof SimpleFSIndexInput) { + // flush any bytes in the buffer + flush(); + // do the optimized copy + FileChannel in = ((SimpleFSIndexInput) input).file.getChannel(); + FileChannel out = file.getChannel(); + copy(in, out, numBytes); + // corrects the position in super (BufferedIndexOutput), so that calls + // to getFilePointer will return the correct pointer. + // Perhaps a specific method is better? + super.seek(out.position()); + } else { super.copyBytes(input, numBytes); - return; } - - SimpleFSIndexInput fsInput = (SimpleFSIndexInput) input; - - // flush any bytes in the input's buffer. - numBytes -= fsInput.flushBuffer(this, numBytes); - - // flush any bytes in the buffer - flush(); - - // do the optimized copy - FileChannel in = fsInput.file.getChannel(); - - // Necessary because BufferedIndexInput does lazy seeking: - in.position(fsInput.getFilePointer()); - - FileChannel out = file.getChannel(); - long pos = out.position(); - long writeTo = numBytes + pos; - while (pos < writeTo) { - pos += out.transferFrom(in, pos, Math.min(CHANNEL_CHUNK_SIZE, writeTo - pos)); - } - // transferFrom does not change the position of the channel. Need to change it manually - out.position(pos); - - // corrects the position in super (BufferedIndexOutput), so that calls - // to getFilePointer will return the correct pointer. - // Perhaps a specific method is better? - super.seek(out.position()); } @Override