Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 688738) +++ CHANGES.txt (working copy) @@ -114,6 +114,7 @@ hashCode() and equals() in Token, and fixed all core and contrib analyzers to use the re-use APIs. (DM Smith via Mike McCandless) +<<<<<<< .mine 18. LUCENE-1329: Add optional readOnly boolean when opening an IndexReader. A readOnly reader is not allowed to make changes (deletions, norms) to the index; in exchanged, the isDeleted @@ -122,6 +123,16 @@ 3.0 the default will become true. (Jason Rutherglen via Mike McCandless) +======= +18. LUCENE-753: Added new Directory implementation + org.apache.lucene.store.NIOFSDirectory, which uses java nio's + FileChannel to do file reads. On most non-Windows platforms, with + many threads sharing a single searcher, this may yield sizable + query throughput when compared to FSDirectory which only allows a + single thread to read from an open file at a time. (Jason + Rutherglen via Mike McCandless) + +>>>>>>> .theirs Bug fixes 1. LUCENE-1134: Fixed BooleanQuery.rewrite to only optimize a single Index: src/java/org/apache/lucene/store/BufferedIndexInput.java =================================================================== --- src/java/org/apache/lucene/store/BufferedIndexInput.java (revision 688738) +++ src/java/org/apache/lucene/store/BufferedIndexInput.java (working copy) @@ -25,13 +25,13 @@ /** Default buffer size */ public static final int BUFFER_SIZE = 1024; - private int bufferSize = BUFFER_SIZE; + protected int bufferSize = BUFFER_SIZE; - private byte[] buffer; + protected byte[] buffer; - private long bufferStart = 0; // position in file of buffer - private int bufferLength = 0; // end of valid bytes - private int bufferPosition = 0; // next byte to read + protected long bufferStart = 0; // position in file of buffer + protected int bufferLength = 0; // end of valid bytes + protected int bufferPosition = 0; // next byte to read public byte readByte() throws IOException { if (bufferPosition >= bufferLength) Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 688738) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -544,7 +544,7 @@ private static class Descriptor extends RandomAccessFile { // remember if the file is open, so that we don't try to close it // more than once - private boolean isOpen; + private volatile boolean isOpen; long position; final long length; @@ -633,7 +633,7 @@ // remember if the file is open, so that we don't try to close it // more than once - private boolean isOpen; + private volatile boolean isOpen; public FSIndexOutput(File path) throws IOException { file = new RandomAccessFile(path, "rw"); Index: src/java/org/apache/lucene/store/NIOFSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 0) +++ src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 0) @@ -0,0 +1,213 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +/** + * NIO version of FSDirectory. Uses FileChannel.read(ByteBuffer dst, long position) method + * which allows multiple threads to read from the file without synchronizing. FSDirectory + * synchronizes in the FSIndexInput.readInternal method which can cause slowness when there + * are many threads accessing the Directory concurrently. + * + * Note: NIOFSDirectory is not recommended on Windows because of a bug + * in how FileChannel.read is implemented. Inside of the implementation + * the position is synchronized. + * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734 + * + * @see FSDirectory + */ +public class NIOFSDirectory extends FSDirectory { + + // Inherit javadoc + public IndexInput openInput(String name, int bufferSize) throws IOException { + ensureOpen(); + return new NIOFSIndexInput(new File(getFile(), name), bufferSize); + } + + protected static class NIOFSIndexInput extends BufferedIndexInput { + + private static class Descriptor extends RandomAccessFile { + // remember if the file is open, so that we don't try to close it + // more than once + private volatile boolean isOpen; + long position; + final long length; + final FileChannel channel; + + public Descriptor(File file, String mode) throws IOException { + super(file, mode); + isOpen = true; + length = length(); + channel = this.getChannel(); + } + + public void close() throws IOException { + if (isOpen) { + isOpen = false; + super.close(); + } + } + + protected void finalize() throws Throwable { + try { + close(); + } finally { + super.finalize(); + } + } + } + + private final Descriptor file; + boolean isClone; + private ByteBuffer bytebuf; // wraps the buffer for NIO + + public NIOFSIndexInput(File path) throws IOException { + this(path, BufferedIndexInput.BUFFER_SIZE); + } + + public NIOFSIndexInput(File path, int bufferSize) throws IOException { + file = new Descriptor(path, "r"); + } + + public void readBytes(byte[] b, int offset, int len) throws IOException { + if (len <= (bufferLength - bufferPosition)) { + // the buffer contains enough data to satistfy this request + if (len > 0) // to allow b to be null if len is 0... + System.arraycopy(buffer, bufferPosition, b, offset, len); + bufferPosition += len; + } else { + // the buffer does not have enough data. First serve all we've got. + int available = bufferLength - bufferPosition; + if (available > 0) { + System.arraycopy(buffer, bufferPosition, b, offset, available); + offset += available; + len -= available; + bufferPosition += available; + } + + // and now, read the remaining 'len' bytes: + if (len < BUFFER_SIZE) { + // If the amount left to read is small enough, do it in the usual + // buffered way: fill the buffer and copy from it: + refill(); + if (bufferLength < len) { + // Throw an exception when refill() could not read len bytes: + System.arraycopy(buffer, 0, b, offset, bufferLength); + throw new IOException("read past EOF"); + + } else { + System.arraycopy(buffer, 0, b, offset, len); + bufferPosition = len; + } + } else { + // The amount left to read is larger than the buffer - there's no + // performance reason not to read it all at once. Note that unlike + // the previous code of this function, there is no need to do a seek + // here, because there's no need to reread what we had in the buffer. + long after = bufferStart + bufferPosition + len; + readInternal(b, offset, len); + bufferStart = after; + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read + } + } + } + + private void read(ByteBuffer bb) throws IOException { + long pos = getFilePointer(); + while (bb.hasRemaining()) { + int i = file.channel.read(bb, pos); + if (i == -1) + throw new IOException("read past EOF"); + pos += i; + } + } + + private void refill() throws IOException { + long start = bufferStart + bufferPosition; + long end = start + BUFFER_SIZE; + if (end > length()) + end = length(); + int newLength = (int) (end - start); + if (newLength <= 0) + throw new IOException("read past EOF"); + + if (buffer == null) { + buffer = new byte[BUFFER_SIZE]; // allocate buffer lazily + bytebuf = ByteBuffer.wrap(buffer); + } else { + bytebuf.clear(); + } + + // since we try to read a complete buffer, we need to limit it if + // it's past the end of the file. + if (newLength != BUFFER_SIZE) + bytebuf.limit(newLength); + + read(bytebuf); + + bufferLength = newLength; + bufferStart = start; + bufferPosition = 0; + } + + /** + * No need to do anything + */ + protected void seekInternal(long pos) throws IOException { + } + + protected void readInternal(byte[] b, int offset, int len) throws IOException { + read(ByteBuffer.wrap(b, offset, len)); + } + + public long length() { + return file.length; + } + + public Object clone() { + NIOFSIndexInput clone = (NIOFSIndexInput) super.clone(); + clone.isClone = true; + if (buffer != null) { + clone.buffer = new byte[buffer.length]; + System.arraycopy(buffer, 0, clone.buffer, 0, bufferLength); + clone.bytebuf = ByteBuffer.wrap(clone.buffer); + } + return clone; + } + + public void close() throws IOException { + // only close the file if this is not a clone + if (!isClone) + file.close(); + } + + /** + * Method used for testing. Returns true if the underlying file descriptor + * is valid. + */ + boolean isFDValid() throws IOException { + return file.getFD().valid(); + } + } +} Index: src/test/org/apache/lucene/store/TestNIOFSDirectory.java =================================================================== --- src/test/org/apache/lucene/store/TestNIOFSDirectory.java (revision 0) +++ src/test/org/apache/lucene/store/TestNIOFSDirectory.java (revision 0) @@ -0,0 +1,77 @@ +package org.apache.lucene.store; + +import java.io.File; +import java.util.HashSet; +import java.util.Random; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.LuceneTestCase; + +public class TestNIOFSDirectory extends LuceneTestCase { + + private final static String alphabet = "abcdefghijklmnopqrstuvwzyz"; + private Random random; + + public void setUp() throws Exception { + super.setUp(); + random = new Random(); + System.setProperty("org.apache.lucene.FSDirectory.class", "org.apache.lucene.store.NIOFSDirectory"); + } + + private String randomToken() { + int tl = 1 + random.nextInt(7); + StringBuffer sb = new StringBuffer(); + for (int cx = 0; cx < tl; cx++) { + int c = random.nextInt(25); + sb.append(alphabet.substring(c, c + 1)); + } + return sb.toString(); + } + + private String randomField() { + int fl = 1 + random.nextInt(3); + StringBuffer fb = new StringBuffer(); + for (int fx = 0; fx < fl; fx++) { + fb.append(randomToken()); + fb.append(" "); + } + return fb.toString(); + } + + private final static String storePathname = new File(System.getProperty("tempDir"), "testLuceneMmap").getAbsolutePath(); + + public void testMmapIndex() throws Exception { + FSDirectory storeDirectory; + storeDirectory = FSDirectory.getDirectory(storePathname); + + // plan to add a set of useful stopwords, consider changing some of the + // interior filters. + StandardAnalyzer analyzer = new StandardAnalyzer(new HashSet()); + // TODO: something about lock timeouts and leftover locks. + IndexWriter writer = new IndexWriter(storeDirectory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); + IndexSearcher searcher = new IndexSearcher(storePathname); + + for (int dx = 0; dx < 1000; dx++) { + String f = randomField(); + Document doc = new Document(); + doc.add(new Field("data", f, Field.Store.YES, Field.Index.TOKENIZED)); + writer.addDocument(doc); + } + + searcher.close(); + writer.close(); + rmDir(new File(storePathname)); + } + + private void rmDir(File dir) { + File[] files = dir.listFiles(); + for (int i = 0; i < files.length; i++) { + files[i].delete(); + } + dir.delete(); + } +} \ No newline at end of file