Index: src/java/org/apache/lucene/index/CompoundFileWriter.java =================================================================== --- src/java/org/apache/lucene/index/CompoundFileWriter.java (revision 609547) +++ src/java/org/apache/lucene/index/CompoundFileWriter.java (working copy) @@ -165,12 +165,11 @@ // Open the files and copy their data into the stream. // Remember the locations of each file's data section. - byte buffer[] = new byte[16384]; it = entries.iterator(); while(it.hasNext()) { FileEntry fe = (FileEntry) it.next(); fe.dataOffset = os.getFilePointer(); - copyFile(fe, os, buffer); + copyFile(fe, os); } // Write the data offsets into the directory of the compound stream @@ -195,10 +194,8 @@ } /** Copy the contents of the file with specified extension into the - * provided output stream. Use the provided buffer for moving data - * to reduce memory allocation. - */ - private void copyFile(FileEntry source, IndexOutput os, byte buffer[]) + * provided output stream. */ + private void copyFile(FileEntry source, IndexOutput os) throws IOException { IndexInput is = null; @@ -206,35 +203,36 @@ long startPtr = os.getFilePointer(); is = directory.openInput(source.file); - long length = is.length(); - long remainder = length; - int chunk = buffer.length; + final long inputSize = is.length(); - while(remainder > 0) { - int len = (int) Math.min(chunk, remainder); - is.readBytes(buffer, 0, len); - os.writeBytes(buffer, len); - remainder -= len; - if (checkAbort != null) - // Roughly every 2 MB we will check if - // it's time to abort - checkAbort.work(80); + final long chunkSize; + if (checkAbort != null) + // Do it (large) chunks so we can check for + // abort after each chunk: + chunkSize = 10*1024*1024; + else + chunkSize = Long.MAX_VALUE; + + long left = inputSize; + while (left > 0) { + final long inc; + if (left > chunkSize) + inc = chunkSize; + else + inc = left; + os.copyBytes(is, inc); + left -= inc; + if (checkAbort != null) + checkAbort.work(10000); } - // Verify that remainder is 0 - if (remainder != 0) - throw new IOException( - "Non-zero remainder length after copying: " + remainder - + " (id: " + source.file + ", length: " + length - + ", buffer size: " + chunk + ")"); - // Verify that the output length diff is equal to original file long endPtr = os.getFilePointer(); long diff = endPtr - startPtr; - if (diff != length) + if (diff != is.length()) throw new IOException( "Difference in the output file offsets " + diff - + " does not match the original file length " + length); + + " does not match the original file length " + is.length()); } finally { if (is != null) is.close(); Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 609547) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -22,6 +22,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Hashtable; @@ -535,6 +536,10 @@ super(bufferSize); file = new Descriptor(path, "r"); } + + RandomAccessFile getFile() { + return file; + } /** IndexInput methods */ protected void readInternal(byte[] b, int offset, int len) @@ -606,6 +611,33 @@ isOpen = false; } } + + public void copyBytes(IndexInput input, long numBytes) throws IOException { + + // If we are copying from an FSIndexInput, and, it's a + // large copy (bigger than our buffer size), then use + // nio.transferTo for faster copy + if (input instanceof FSIndexInput && numBytes > BUFFER_SIZE) { + + flush(); + + FSIndexInput fsInput = (FSIndexInput) input; + + FileChannel in = fsInput.getFile().getChannel(); + FileChannel out = file.getChannel(); + + final long inputPos = input.getFilePointer(); + final long outputPos = getFilePointer(); + long copied = in.transferTo(inputPos, numBytes, out); + if (copied != numBytes) + throw new IOException("failed to copy all bytes: only copied " + copied + " out of " + numBytes); + + seek(outputPos + numBytes); + input.seek(inputPos + numBytes); + + } else + super.copyBytes(input, numBytes); + } /** Random-access methods */ public void seek(long pos) throws IOException {