Index: lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (revision 1458763)
+++ lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (working copy)
@@ -298,8 +298,12 @@
throw new IOException("Cannot overwrite: " + file);
}
- protected void onIndexOutputClosed(FSIndexOutput io) {
- staleFiles.add(io.name);
+ /**
+ * Sub classes should call this method on closing an open {@link IndexOutput}, reporting the name of the file
+ * that was closed. {@code FSDirectory} needs this information to take care of syncing stale files.
+ */
+ protected void onIndexOutputClosed(String name) {
+ staleFiles.add(name);
}
@Override
@@ -392,65 +396,6 @@
return chunkSize;
}
- /** Base class for reading input from a RandomAccessFile */
- protected abstract static class FSIndexInput extends BufferedIndexInput {
- /** the underlying RandomAccessFile */
- protected final RandomAccessFile file;
- boolean isClone = false;
- /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
- protected final int chunkSize;
- /** start offset: non-zero in the slice case */
- protected final long off;
- /** end offset (start+length) */
- protected final long end;
-
- /** Create a new FSIndexInput, reading the entire file from path */
- protected FSIndexInput(String resourceDesc, File path, IOContext context, int chunkSize) throws IOException {
- super(resourceDesc, context);
- this.file = new RandomAccessFile(path, "r");
- this.chunkSize = chunkSize;
- this.off = 0L;
- this.end = file.length();
- }
-
- /** Create a new FSIndexInput, representing a slice of an existing open file */
- protected FSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize, int chunkSize) {
- super(resourceDesc, bufferSize);
- this.file = file;
- this.chunkSize = chunkSize;
- this.off = off;
- this.end = off + length;
- this.isClone = true; // well, we are sorta?
- }
-
- @Override
- public void close() throws IOException {
- // only close the file if this is not a clone
- if (!isClone) {
- file.close();
- }
- }
-
- @Override
- public FSIndexInput clone() {
- FSIndexInput clone = (FSIndexInput)super.clone();
- clone.isClone = true;
- return clone;
- }
-
- @Override
- public final long length() {
- return end - off;
- }
-
- /** Method used for testing. Returns true if the underlying
- * file descriptor is valid.
- */
- boolean isFDValid() throws IOException {
- return file.getFD().valid();
- }
- }
-
/**
* Writes output with {@link RandomAccessFile#write(byte[], int, int)}
*/
@@ -476,7 +421,7 @@
@Override
public void close() throws IOException {
- parent.onIndexOutputClosed(this);
+ parent.onIndexOutputClosed(name);
// only close the file if it has not been closed yet
if (isOpen) {
boolean success = false;
Index: lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java (revision 1458763)
+++ lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java (working copy)
@@ -19,11 +19,11 @@
import java.io.IOException;
import java.io.File;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
+import java.nio.file.StandardOpenOption;
import java.security.AccessController;
import java.security.PrivilegedExceptionAction;
@@ -133,7 +133,7 @@
this.chunkSizePower = 31 - Integer.numberOfLeadingZeros(maxChunkSize);
assert this.chunkSizePower >= 0 && this.chunkSizePower <= 30;
}
-
+
/**
* true, if this platform supports unmapping mmapped files.
*/
@@ -189,12 +189,9 @@
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
- File f = new File(getDirectory(), name);
- RandomAccessFile raf = new RandomAccessFile(f, "r");
- try {
- return new MMapIndexInput("MMapIndexInput(path=\"" + f + "\")", raf);
- } finally {
- raf.close();
+ File file = new File(getDirectory(), name);
+ try (FileChannel c = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
+ return new MMapIndexInput("MMapIndexInput(path=\"" + file.toString() + "\")", c);
}
}
@@ -218,8 +215,8 @@
private final class MMapIndexInput extends ByteBufferIndexInput {
private final boolean useUnmapHack;
- MMapIndexInput(String resourceDescription, RandomAccessFile raf) throws IOException {
- super(resourceDescription, map(raf, 0, raf.length()), raf.length(), chunkSizePower, getUseUnmap());
+ MMapIndexInput(String resourceDescription, FileChannel fc) throws IOException {
+ super(resourceDescription, map(fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap());
this.useUnmapHack = getUseUnmap();
}
@@ -256,9 +253,9 @@
}
/** Maps a file into a set of buffers */
- ByteBuffer[] map(RandomAccessFile raf, long offset, long length) throws IOException {
+ ByteBuffer[] map(FileChannel fc, long offset, long length) throws IOException {
if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
- throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + raf.toString());
+ throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + fc.toString());
final long chunkSize = 1L << chunkSizePower;
@@ -268,13 +265,12 @@
ByteBuffer buffers[] = new ByteBuffer[nrBuffers];
long bufferStart = 0L;
- FileChannel rafc = raf.getChannel();
for (int bufNr = 0; bufNr < nrBuffers; bufNr++) {
int bufSize = (int) ( (length > (bufferStart + chunkSize))
? chunkSize
: (length - bufferStart)
);
- buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
+ buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
bufferStart += bufSize;
}
Index: lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 1458763)
+++ lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java (working copy)
@@ -20,10 +20,10 @@
import java.io.File;
import java.io.EOFException;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
import java.util.concurrent.Future; // javadoc
/**
@@ -77,7 +77,9 @@
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
- return new NIOFSIndexInput(new File(getDirectory(), name), context, getReadChunkSize());
+ File path = new File(getDirectory(), name);
+ FileChannel fc = FileChannel.open(path.toPath(), StandardOpenOption.READ);
+ return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context, getReadChunkSize());
}
@Override
@@ -85,7 +87,7 @@
final IOContext context) throws IOException {
ensureOpen();
final File path = new File(getDirectory(), name);
- final RandomAccessFile descriptor = new RandomAccessFile(path, "r");
+ final FileChannel descriptor = FileChannel.open(path.toPath(), StandardOpenOption.READ);
return new Directory.IndexInputSlicer() {
@Override
@@ -95,7 +97,7 @@
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
- return new NIOFSIndexInput(sliceDescription, path, descriptor, descriptor.getChannel(), offset,
+ return new NIOFSIndexInput("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
length, BufferedIndexInput.bufferSize(context), getReadChunkSize());
}
};
@@ -104,22 +106,55 @@
/**
* Reads bytes with {@link FileChannel#read(ByteBuffer, long)}
*/
- protected static class NIOFSIndexInput extends FSIndexInput {
-
+ protected static class NIOFSIndexInput extends BufferedIndexInput {
+ /** the file channel we will read from */
+ protected final FileChannel channel;
+ /** is this instance a clone and hence does not own the file to close it */
+ boolean isClone = false;
+ /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
+ protected final int chunkSize;
+ /** start offset: non-zero in the slice case */
+ protected final long off;
+ /** end offset (start+length) */
+ protected final long end;
+
private ByteBuffer byteBuf; // wraps the buffer for NIO
- final FileChannel channel;
-
- public NIOFSIndexInput(File path, IOContext context, int chunkSize) throws IOException {
- super("NIOFSIndexInput(path=\"" + path + "\")", path, context, chunkSize);
- channel = file.getChannel();
+ public NIOFSIndexInput(String resourceDesc, FileChannel fc, IOContext context, int chunkSize) throws IOException {
+ super(resourceDesc, context);
+ this.channel = fc;
+ this.chunkSize = chunkSize;
+ this.off = 0L;
+ this.end = fc.size();
}
- public NIOFSIndexInput(String sliceDescription, File path, RandomAccessFile file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) {
- super("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + off + ":" + (off+length) + ")", file, off, length, bufferSize, chunkSize);
- channel = fc;
- isClone = true;
+ public NIOFSIndexInput(String resourceDesc, FileChannel fc, long off, long length, int bufferSize, int chunkSize) {
+ super(resourceDesc, bufferSize);
+ this.channel = fc;
+ this.chunkSize = chunkSize;
+ this.off = off;
+ this.end = off + length;
+ this.isClone = true;
}
+
+ @Override
+ public void close() throws IOException {
+ if (!isClone) {
+ channel.close();
+ }
+ }
+
+ @Override
+ public NIOFSIndexInput clone() {
+ NIOFSIndexInput clone = (NIOFSIndexInput)super.clone();
+ clone.isClone = true;
+ return clone;
+ }
+
+ @Override
+ public final long length() {
+ return end - off;
+ }
@Override
protected void newBuffer(byte[] newBuffer) {
@@ -186,5 +221,4 @@
@Override
protected void seekInternal(long pos) throws IOException {}
}
-
}
Index: lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 1458763)
+++ lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java (working copy)
@@ -55,7 +55,8 @@
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
final File path = new File(directory, name);
- return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", path, context, getReadChunkSize());
+ RandomAccessFile raf = new RandomAccessFile(path, "r");
+ return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context, getReadChunkSize());
}
@Override
@@ -83,15 +84,53 @@
* Reads bytes with {@link RandomAccessFile#seek(long)} followed by
* {@link RandomAccessFile#read(byte[], int, int)}.
*/
- protected static class SimpleFSIndexInput extends FSIndexInput {
-
- public SimpleFSIndexInput(String resourceDesc, File path, IOContext context, int chunkSize) throws IOException {
- super(resourceDesc, path, context, chunkSize);
+ protected static class SimpleFSIndexInput extends BufferedIndexInput {
+ /** the file channel we will read from */
+ protected final RandomAccessFile file;
+ /** is this instance a clone and hence does not own the file to close it */
+ boolean isClone = false;
+ /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
+ protected final int chunkSize;
+ /** start offset: non-zero in the slice case */
+ protected final long off;
+ /** end offset (start+length) */
+ protected final long end;
+
+ public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, IOContext context, int chunkSize) throws IOException {
+ super(resourceDesc, context);
+ this.file = file;
+ this.chunkSize = chunkSize;
+ this.off = 0L;
+ this.end = file.length();
}
public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize, int chunkSize) {
- super(resourceDesc, file, off, length, bufferSize, chunkSize);
+ super(resourceDesc, bufferSize);
+ this.file = file;
+ this.chunkSize = chunkSize;
+ this.off = off;
+ this.end = off + length;
+ this.isClone = true;
}
+
+ @Override
+ public void close() throws IOException {
+ if (!isClone) {
+ file.close();
+ }
+ }
+
+ @Override
+ public SimpleFSIndexInput clone() {
+ SimpleFSIndexInput clone = (SimpleFSIndexInput)super.clone();
+ clone.isClone = true;
+ return clone;
+ }
+
+ @Override
+ public final long length() {
+ return end - off;
+ }
/** IndexInput methods */
@Override
@@ -136,5 +175,9 @@
@Override
protected void seekInternal(long position) {
}
+
+ boolean isFDValid() throws IOException {
+ return file.getFD().valid();
+ }
}
}
Index: lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (revision 1458763)
+++ lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (working copy)
@@ -21,6 +21,9 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -93,16 +96,16 @@
writeBytes(tmpInputFile, TEST_FILE_LENGTH);
// run test with chunk size of 10 bytes
- runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")", tmpInputFile,
- newIOContext(random()), 10), inputBufferSize, random());
+ runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")",
+ new RandomAccessFile(tmpInputFile, "r"), newIOContext(random()), 10), inputBufferSize, random());
// run test with chunk size of 10 bytes
- runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
- newIOContext(random()), 10), inputBufferSize, random());
+ runReadBytesAndClose(new NIOFSIndexInput("NIOFSIndexInput(path=\"" + tmpInputFile + "\")",
+ FileChannel.open(tmpInputFile.toPath(), StandardOpenOption.READ), newIOContext(random()), 10),
+ inputBufferSize, random());
}
- private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
- throws IOException {
+ private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r) throws IOException {
try {
runReadBytes(input, bufferSize, r);
} finally {