Index: lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (revision 1142967) +++ lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (working copy) @@ -27,6 +27,7 @@ import org.apache.lucene.store.Directory; // javadoc import org.apache.lucene.store.NativeFSLockFactory; // javadoc +import org.apache.lucene.store.IOContext.Context; /** * An {@link Directory} implementation that uses the @@ -72,14 +73,19 @@ public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); //nocommit - use buffer based on IOContext - return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexInput.BUFFER_SIZE : forcedBufferSize); + if (forcedBufferSize != 0) + return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize, context); + + else if (context.context == Context.MERGE) + return new DirectIOLinuxIndexInput(new File(getDirectory(), name), BufferedIndexInput.MERGE_BUFFER_SIZE, context); + + return new DirectIOLinuxIndexInput(new File(getDirectory(), name), BufferedIndexInput.BUFFER_SIZE, context); } @Override - public IndexOutput createOutput(String name,IOContext context) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); ensureCanWrite(name); - //nocommit - use buffer based on IOContext return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexOutput.BUFFER_SIZE : forcedBufferSize); } @@ -239,8 +245,14 @@ private long filePos; private int bufferPos; - public DirectIOLinuxIndexInput(File path, int bufferSize) throws IOException { - FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true); + public DirectIOLinuxIndexInput(File path, int bufferSize, IOContext context) throws IOException { + FileDescriptor fd; + + if (context.context == Context.MERGE) + fd = NativePosixUtil.open_direct(path.toString(), true); + else + fd = NativePosixUtil.open_normal(path.toString(), true); + fis = new FileInputStream(fd); channel = fis.getChannel(); this.bufferSize = bufferSize; Index: lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.java (revision 1142967) +++ lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.java (working copy) @@ -37,6 +37,7 @@ public static native int posix_madvise(ByteBuffer buf, int advise) throws IOException; public static native int madvise(ByteBuffer buf, int advise) throws IOException; public static native FileDescriptor open_direct(String filename, boolean read) throws IOException; + public static native FileDescriptor open_normal(String filename, boolean read) throws IOException; public static native long pread(FileDescriptor fd, long pos, ByteBuffer byteBuf) throws IOException; public static void advise(FileDescriptor fd, long offset, long len, int advise) throws IOException { Index: lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp (revision 1142967) +++ lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp (working copy) @@ -16,14 +16,26 @@ */ #include -#include // posix_fadvise, constants for open +#include // posix_fadvise, constants for open, for mac os x #include // strerror #include // errno #include // pread #include // posix_madvise, madvise #include // constants for open #include // constants for open +#include +#ifdef _WIN32 // For future implementation + #define OS "win32" +#elif linux + #define OS "linux" +#elif __APPLE__ + #define OS "osx" +#else + #define OS "bsd" // __unix__ is not used as even Linux falls under it. +#endif + + // java -cp .:lib/junit-4.7.jar:./build/classes/test:./build/classes/java:./build/classes/demo -Dlucene.version=2.9-dev -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.TestDoc /* @@ -113,12 +125,26 @@ fname = (char *) env->GetStringUTFChars(filename, NULL); - if (readOnly) { - fd = open(fname, O_RDONLY | O_DIRECT | O_NOATIME); - } else { - fd = open(fname, O_RDWR | O_CREAT | O_DIRECT | O_NOATIME, 0666); + if ( strcmp(OS, "linux") || strcmp(OS,"bsd")) { + if (readOnly) { + fd = open(fname, O_RDONLY | O_DIRECT | O_NOATIME); + } + else { + fd = open(fname, O_RDWR | O_CREAT | O_DIRECT | O_NOATIME, 0666); + } } + else if ( strcmp(OS, "osx")) { + if (readOnly) { + fd = open(fname, O_RDONLY | O_NOATIME); + fcntl(fd, F_NOCACHE, 1); + } + else { + fd = open(fname, O_RDWR | O_CREAT | O_NOATIME, 0666); + fcntl(fd, F_NOCACHE, 1); + } + } + //printf("open %s -> %d; ro %d\n", fname, fd, readOnly); fflush(stdout); env->ReleaseStringUTFChars(filename, fname); @@ -146,6 +172,60 @@ /* * Class: org_apache_lucene_store_NativePosixUtil + * Method: open_normal + * Signature: (Ljava/lang/String;Z)Ljava/io/FileDescriptor; + */ +extern "C" +JNIEXPORT jobject JNICALL Java_org_apache_lucene_store_NativePosixUtil_open_1normal(JNIEnv *env, jclass _ignore, jstring filename, jboolean readOnly) +{ + jfieldID field_fd; + jmethodID const_fdesc; + jclass class_fdesc, class_ioex; + jobject ret; + int fd; + char *fname; + + class_ioex = env->FindClass("java/io/IOException"); + if (class_ioex == NULL) return NULL; + class_fdesc = env->FindClass("java/io/FileDescriptor"); + if (class_fdesc == NULL) return NULL; + + fname = (char *) env->GetStringUTFChars(filename, NULL); + + if (readOnly) { + fd = open(fname, O_RDONLY | O_NOATIME); + } + else { + fd = open(fname, O_RDWR | O_CREAT | O_NOATIME, 0666); + } + + //printf("open %s -> %d; ro %d\n", fname, fd, readOnly); fflush(stdout); + + env->ReleaseStringUTFChars(filename, fname); + + if (fd < 0) { + // open returned an error. Throw an IOException with the error string + env->ThrowNew(class_ioex, strerror(errno)); + return NULL; + } + + // construct a new FileDescriptor + const_fdesc = env->GetMethodID(class_fdesc, "", "()V"); + if (const_fdesc == NULL) return NULL; + ret = env->NewObject(class_fdesc, const_fdesc); + + // poke the "fd" field with the file descriptor + field_fd = env->GetFieldID(class_fdesc, "fd", "I"); + if (field_fd == NULL) return NULL; + env->SetIntField(ret, field_fd, fd); + + // and return it + return ret; +} + + +/* + * Class: org_apache_lucene_store_NativePosixUtil * Method: pread * Signature: (Ljava/io/FileDescriptor;JLjava/nio/ByteBuffer;)I */ @@ -257,6 +337,72 @@ /* * Class: org_apache_lucene_store_NativePosixUtil + * Method: posix_madvise + * Signature: (Ljava/nio/ByteBuffer;I)I + */ +extern "C" +JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_1madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) { + void *p = env->GetDirectBufferAddress(buffer); + if (p == NULL) { + return -1; + } + + size_t size = (size_t) env->GetDirectBufferCapacity(buffer); + if (size <= 0) { + return -1; + } + + int page = getpagesize(); + + // round start down to start of page + long long start = (long long) p; + start = start & (~(page-1)); + + // round end up to start of page + long long end = start + size; + end = (end + page-1)&(~(page-1)); + size = (end-start); + + int osAdvice; + switch(advice) { + case 0: + osAdvice = MADV_NORMAL; + break; + case 1: + osAdvice = MADV_SEQUENTIAL; + break; + case 2: + osAdvice = MADV_RANDOM; + break; + case 3: + osAdvice = MADV_WILLNEED; + break; + case 4: + osAdvice = MADV_DONTNEED; + break; + case 5: + return -1; + break; + } + + //printf("DO madvise: %lx %d\n", p, size);fflush(stdout); + + if (madvise((void *) start, size, osAdvice) != 0) { + jclass class_ioex = env->FindClass("java/io/IOException"); + if (class_ioex == NULL) { + return -1; + } + + env->ThrowNew(class_ioex, strerror(errno)); + return -1; + } + + return 0; +} + + +/* + * Class: org_apache_lucene_store_NativePosixUtil * Method: madvise * Signature: (Ljava/nio/ByteBuffer;I)I */ Index: lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java =================================================================== --- lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (revision 1142967) +++ lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (working copy) @@ -275,7 +275,7 @@ // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); //nocommit is IOContext.DEFAULT the right thing to do here? - SegmentReader reader = readerPool.get(info, false, IOContext.DEFAULT); + SegmentReader reader = readerPool.get(info, false, IOContext.READ); int delCount = 0; final boolean segAllDeletes; try { Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (revision 1142967) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ChecksumIndexOutput; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; @@ -57,8 +58,7 @@ @Override public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) throws IOException { - //nocommit should this context always be flush? - IndexOutput out = createOutput(dir, segmentFileName, context); + IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount()))); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT Index: lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java =================================================================== --- lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java (revision 1142967) +++ lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java (working copy) @@ -24,6 +24,14 @@ /** Default buffer size */ public static final int BUFFER_SIZE = 1024; + + // The normal read buffer size defaults to 1024, but + // increasing this during merging seems to yield + // performance gains. However we don't want to increase + // it too much because there are quite a few + // BufferedIndexInputs created during merging. See + // LUCENE-888 for details. + public static final int MERGE_BUFFER_SIZE = 1024; private int bufferSize = BUFFER_SIZE; Index: lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 1142967) +++ lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (working copy) @@ -21,7 +21,9 @@ import java.io.IOException; import java.io.RandomAccessFile; +import org.apache.lucene.store.IOContext.Context; + /** A straightforward implementation of {@link FSDirectory} * using java.io.RandomAccessFile. However, this class has * poor concurrent performance (multiple threads will @@ -89,6 +91,9 @@ public SimpleFSIndexInput(File path, IOContext context, int chunkSize) throws IOException { //nocommit Use IOContext to decide bufferSize instead of BufferedIndexInput.BUFFER_SIZE super(BufferedIndexInput.BUFFER_SIZE); + if (context.context == Context.MERGE) { + setBufferSize(MERGE_BUFFER_SIZE); + } file = new Descriptor(path, "r"); this.chunkSize = chunkSize; } Index: lucene/src/java/org/apache/lucene/store/MergeInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/store/MergeInfo.java (revision 1138659) +++ lucene/src/java/org/apache/lucene/store/MergeInfo.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.index; +package org.apache.lucene.store; /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,7 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -//nocommit javadoc + +/** + *

A MergeInfo provides information required for a MERGE context and other optimization operations. + * It is used as part of an {@link IOContext} in case of MERGE context.

+ */ + public class MergeInfo { public int totalDocCount; @@ -25,7 +30,16 @@ boolean isExternal; // used by IndexWriter boolean optimize; // used by IndexWriter + + /** + *

Creates a new {@link MergeInfo} instance from + * the values required for a MERGE {@link IOContext} context. + * + * These values are only estimates and are not the actual values. + * + */ + public MergeInfo(int totalDocCount, long estimatedMergeBytes, boolean isExternal, boolean optimize) { this.totalDocCount = totalDocCount; this.estimatedMergeBytes = estimatedMergeBytes; Index: lucene/src/java/org/apache/lucene/store/IOContext.java =================================================================== --- lucene/src/java/org/apache/lucene/store/IOContext.java (revision 1138659) +++ lucene/src/java/org/apache/lucene/store/IOContext.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.index; +package org.apache.lucene.store; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -37,6 +37,8 @@ public final MergeInfo mergeInfo; + public final FlushInfo flushInfo; + public final boolean readOnce; public static final IOContext DEFAULT = new IOContext(Context.DEFAULT); @@ -49,14 +51,23 @@ this(false); } - public IOContext(Context context) { + public IOContext (FlushInfo flushInfo) { + assert flushInfo != null; + this.context = Context.FLUSH; + this.mergeInfo = null; + this.readOnce = false; + this.flushInfo = flushInfo; + } + + public IOContext (Context context) { this(context, null); } - private IOContext(boolean readOnce) { + private IOContext (boolean readOnce) { this.context = Context.READ; this.mergeInfo = null; this.readOnce = readOnce; + this.flushInfo = null; } public IOContext (MergeInfo mergeInfo) { @@ -68,6 +79,7 @@ this.context = context; this.readOnce = false; this.mergeInfo = mergeInfo; + this.flushInfo = null; } } \ No newline at end of file