Index: lucene/contrib/misc/build.xml --- lucene/contrib/misc/build.xml Wed Jun 15 05:22:04 2011 -0400 +++ lucene/contrib/misc/build.xml Sat Jun 18 12:23:31 2011 -0400 @@ -59,9 +59,11 @@ + + Index: lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp --- lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp Wed Jun 15 05:22:04 2011 -0400 +++ lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp Sat Jun 18 12:23:31 2011 -0400 @@ -24,6 +24,8 @@ #include // constants for open #include // constants for open +// nocommit -- move to oal.util? + // java -cp .:lib/junit-4.7.jar:./build/classes/test:./build/classes/java:./build/classes/demo -Dlucene.version=2.9-dev -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.TestDoc /* @@ -320,3 +322,91 @@ return 0; } + +/* + * Class: org_apache_lucene_store_NativePosixUtil + * Method: mlock + * Signature: (Ljava/nio/ByteBuffer;II)I + */ +extern "C" +JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_mlock(JNIEnv *env, jclass _ignore, jobject buffer, jint offset, jint len) { + void *p = env->GetDirectBufferAddress(buffer); + if (p == NULL) { + return -1; + } + + size_t size = (size_t) env->GetDirectBufferCapacity(buffer); + if (size <= 0) { + return -1; + } + + int page = getpagesize(); + + // round start down to start of page + long long start = ((long long) p) + offset; + start = start & (~(page-1)); + + // round end up to start of page + long long end = ((long long) p) + offset + len; + end = (end + page-1)&(~(page-1)); + size = (end-start); + + printf("DO mlock: page=%d start=0x%lx end=0x%lx\n", page, start, end);fflush(stdout); + + if (mlock((void *) start, size) != 0) { + printf(" errno=%d\n", errno);fflush(stdout); + jclass class_ioex = env->FindClass("java/io/IOException"); + if (class_ioex == NULL) { + return -1; + } + + env->ThrowNew(class_ioex, strerror(errno)); + return -1; + } + + return 0; +} + + +/* + * Class: org_apache_lucene_store_NativePosixUtil + * Method: munlock + * Signature: (Ljava/nio/ByteBuffer;II)I + */ +extern "C" +JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_munlock(JNIEnv *env, jclass _ignore, jobject buffer, jint offset, jint len) { + void *p = env->GetDirectBufferAddress(buffer); + if (p == NULL) { + return -1; + } + + size_t size = (size_t) env->GetDirectBufferCapacity(buffer); + if (size <= 0) { + return -1; + } + + int page = getpagesize(); + + // round start down to start of page + long long start = ((long long) p) + offset; + start = start & (~(page-1)); + + // round end up to start of page + long long end = ((long long) p) + offset + len; + end = (end + page-1)&(~(page-1)); + size = (end-start); + + printf("DO munlock: page=%d start=0x%lx end=0x%lx\n", page, start, end);fflush(stdout); + + if (munlock((void *) start, size) != 0) { + jclass class_ioex = env->FindClass("java/io/IOException"); + if (class_ioex == NULL) { + return -1; + } + printf(" errno=%d\n", errno);fflush(stdout); + env->ThrowNew(class_ioex, strerror(errno)); + return -1; + } + + return 0; +} Index: lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.java --- lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.java Wed Jun 15 05:22:04 2011 -0400 +++ lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.java Sat Jun 18 12:23:31 2011 -0400 @@ -17,10 +17,16 @@ * limitations under the License. */ +import java.io.FileDescriptor; import java.io.IOException; -import java.io.FileDescriptor; import java.nio.ByteBuffer; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.codecs.BlockTermsReader; + public final class NativePosixUtil { public final static int NORMAL = 0; public final static int SEQUENTIAL = 1; @@ -36,6 +42,8 @@ private static native int posix_fadvise(FileDescriptor fd, long offset, long len, int advise) throws IOException; public static native int posix_madvise(ByteBuffer buf, int advise) throws IOException; public static native int madvise(ByteBuffer buf, int advise) throws IOException; + public static native int mlock(ByteBuffer buf, int offset, int len) throws IOException; + public static native int munlock(ByteBuffer buf, int offset, int len) throws IOException; public static native FileDescriptor open_direct(String filename, boolean read) throws IOException; public static native long pread(FileDescriptor fd, long pos, ByteBuffer byteBuf) throws IOException; @@ -45,5 +53,68 @@ throw new RuntimeException("posix_fadvise failed code=" + code); } } + + // NOTE: + // - had to increase max locked mem allowed for users + // in /etc/security/limits.conf + // - then ulimit -l unlimited + public static void mlockTermsDict(IndexReader reader, String field) throws IOException { + System.out.println("mlockTermsDict reader=" + reader + " field=" + field); + final IndexReader[] subs; + if (reader instanceof SegmentReader) { + subs = new IndexReader[] {reader}; + } else { + subs = reader.getSequentialSubReaders(); + } + for (IndexReader subReader : subs) { + SegmentReader segReader = (SegmentReader) subReader; + System.out.println(" r=" + segReader); + Fields fields = segReader.fields(); + if (fields == null) { + continue; + } + Terms terms = fields.terms(field); + if (terms == null) { + continue; + } + + BlockTermsReader blockTerms = terms.getBlockTermsReader(); + + System.out.println(" got terms=" + terms); + if (blockTerms != null) { + System.out.println(" got BlockTermsReader"); + BlockTermsReader termsReader = blockTerms; + + BlockTermsReader.FileRefSlice slice = termsReader.getFieldFileSlice(field); + + if (slice != null) { + System.out.println(" slice: offset=" + slice.offset + " length=" + slice.length); + + final long sliceStart = slice.offset; + final long sliceEnd = slice.offset + slice.length; + + IndexInput termsInput = termsReader.getTermsIndexInput(); + ByteBuffer[] buffers = termsInput.getByteBuffers(); + if (buffers != null && buffers[0].isDirect()) { + System.out.println(" got " + buffers.length + " buffers"); + long bufferStart = 0; + for(ByteBuffer buffer : buffers) { + long bufferEnd = bufferStart + buffer.capacity(); + if (bufferStart >= sliceEnd || sliceStart >= bufferEnd) { + // no overlap + } else { + long overlapStart = Math.max(bufferStart, sliceStart); + long overlapEnd = Math.min(bufferEnd, sliceEnd); + assert overlapEnd > overlapStart; + System.out.println(" mlock: start=" + overlapStart + " end=" + overlapEnd); + mlock(buffer, (int) (overlapStart - bufferStart), (int) (overlapEnd - overlapStart)); + } + bufferStart = bufferEnd; + } + } + } + } + } + } } Index: lucene/src/java/org/apache/lucene/index/Terms.java --- lucene/src/java/org/apache/lucene/index/Terms.java Wed Jun 15 05:22:04 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/Terms.java Sat Jun 18 12:23:31 2011 -0400 @@ -19,6 +19,8 @@ import java.io.IOException; import java.util.Comparator; + +import org.apache.lucene.index.codecs.BlockTermsReader; // nocommit import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CloseableThreadLocal; @@ -158,4 +160,9 @@ threadEnums.close(); } public final static Terms[] EMPTY_ARRAY = new Terms[0]; + + // nocommit + public BlockTermsReader getBlockTermsReader() { + return null; + } } Index: lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java --- lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java Wed Jun 15 05:22:04 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java Sat Jun 18 12:23:31 2011 -0400 @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.Iterator; import java.util.TreeMap; +import java.util.SortedMap; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; @@ -150,6 +151,47 @@ this.indexReader = indexReader; } + // nocommit + public IndexInput getTermsIndexInput() { + return in; + } + + // nocommit + public final static class FileRefSlice { + public final long offset; + public final long length; + + public FileRefSlice(long offset, long length) { + this.offset = offset; + this.length = length; + } + } + + // nocommit + public FileRefSlice getFieldFileSlice(String field) { + final SortedMap tail = fields.tailMap(field); + String firstField = tail.firstKey(); + + if (firstField.equals(field)) { + final FieldReader first = fields.get(field); + final long start = first.termsStartPointer; + final Iterator iter = tail.values().iterator(); + assert iter.hasNext(); + final FieldReader f1 = iter.next(); + assert f1 == first; + final FieldReader f2 = iter.hasNext() ? iter.next() : null; + final long length; + if (f2 != null) { + length = f2.termsStartPointer - start; + } else { + length = in.length() - start; + } + return new FileRefSlice(start, length); + } else { + return null; + } + } + protected void readHeader(IndexInput input) throws IOException { CodecUtil.checkHeader(input, BlockTermsWriter.CODEC_NAME, BlockTermsWriter.VERSION_START, @@ -254,6 +296,12 @@ this.sumTotalTermFreq = sumTotalTermFreq; } + // nocommit + @Override + public BlockTermsReader getBlockTermsReader() { + return BlockTermsReader.this; + } + @Override public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); Index: lucene/src/java/org/apache/lucene/search/FieldDoc.java --- lucene/src/java/org/apache/lucene/search/FieldDoc.java Wed Jun 15 05:22:04 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/FieldDoc.java Sat Jun 18 12:23:31 2011 -0400 @@ -57,7 +57,7 @@ super (doc, score); this.fields = fields; } - + // A convenience method for debugging. @Override public String toString() { Index: lucene/src/java/org/apache/lucene/store/IndexInput.java --- lucene/src/java/org/apache/lucene/store/IndexInput.java Wed Jun 15 05:22:04 2011 -0400 +++ lucene/src/java/org/apache/lucene/store/IndexInput.java Sat Jun 18 12:23:31 2011 -0400 @@ -20,6 +20,8 @@ import java.io.Closeable; import java.io.IOException; +import java.nio.ByteBuffer; + /** Abstract base class for input from a file in a {@link Directory}. A * random-access input stream. Used for all Lucene index input operations. * @see Directory @@ -45,6 +47,11 @@ /** The number of bytes in the file. */ public abstract long length(); + // nocommit + public ByteBuffer[] getByteBuffers() { + return null; + } + /** * Copies numBytes bytes to the given {@link IndexOutput}. *

Index: lucene/src/java/org/apache/lucene/store/MMapDirectory.java --- lucene/src/java/org/apache/lucene/store/MMapDirectory.java Wed Jun 15 05:22:04 2011 -0400 +++ lucene/src/java/org/apache/lucene/store/MMapDirectory.java Sat Jun 18 12:23:31 2011 -0400 @@ -21,6 +21,7 @@ import java.io.File; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; import java.nio.BufferUnderflowException; import java.nio.channels.ClosedChannelException; // javadoc @link import java.nio.channels.FileChannel; @@ -213,7 +214,7 @@ File f = new File(getDirectory(), name); RandomAccessFile raf = new RandomAccessFile(f, "r"); try { - return new MMapIndexInput(raf, chunkSizePower); + return new MMapIndexInput(raf, chunkSizePower, name); } finally { raf.close(); } @@ -235,7 +236,7 @@ private boolean isClone = false; - MMapIndexInput(RandomAccessFile raf, int chunkSizePower) throws IOException { + MMapIndexInput(RandomAccessFile raf, int chunkSizePower, String name) throws IOException { this.length = raf.length(); this.chunkSizePower = chunkSizePower; this.chunkSize = 1L << chunkSizePower; @@ -262,10 +263,20 @@ : (length - bufferStart) ); this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, bufferStart, bufSize); + // nocommit + //if (name.endsWith("_1.tib")) { + //System.out.println("FORCE LOAD " + name); + //((MappedByteBuffer) this.buffers[bufNr]).load(); + //} bufferStart += bufSize; } seek(0L); } + + // nocommit + public ByteBuffer[] getByteBuffers() { + return buffers; + } @Override public byte readByte() throws IOException {