Index: lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (revision 1143375) +++ lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (working copy) @@ -71,17 +71,21 @@ @Override public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - //nocommit - use buffer based on IOContext - return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexInput.BUFFER_SIZE : forcedBufferSize); + return new DirectIOLinuxIndexInput(new File(getDirectory(), name), + bufferSize(context)); } @Override - public IndexOutput createOutput(String name,IOContext context) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); ensureCanWrite(name); - //nocommit - use buffer based on IOContext - return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexOutput.BUFFER_SIZE : forcedBufferSize); + return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), bufferSize(context)); } + + private int bufferSize(IOContext context) { + return forcedBufferSize != 0 ? forcedBufferSize : BufferedIndexInput + .bufferSize(context); + } private final static class DirectIOLinuxIndexOutput extends IndexOutput { private final ByteBuffer buffer; @@ -240,6 +244,7 @@ private int bufferPos; public DirectIOLinuxIndexInput(File path, int bufferSize) throws IOException { + // TODO make use of IOContext FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true); fis = new FileInputStream(fd); channel = fis.getChannel(); Index: lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (revision 1143375) +++ lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (working copy) @@ -70,7 +70,6 @@ @Override public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - //nocommit - use buffer based on IOContext return new WindowsIndexInput(new File(getDirectory(), name), DEFAULT_BUFFERSIZE); } Index: lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java =================================================================== --- lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (working copy) @@ -274,8 +274,7 @@ if (coalescedDeletes != null) { // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); - //nocommit is IOContext.DEFAULT the right thing to do here? - SegmentReader reader = readerPool.get(info, false, IOContext.DEFAULT); + SegmentReader reader = readerPool.get(info, false, IOContext.READ); int delCount = 0; final boolean segAllDeletes; try { Index: lucene/src/java/org/apache/lucene/index/CompoundFileReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CompoundFileReader.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/index/CompoundFileReader.java (working copy) @@ -143,8 +143,7 @@ final FileEntry entry = entries.get(id); if (entry == null) throw new IOException("No sub-file with id " + id + " found (files: " + entries.keySet() + ")"); - // nocommit set read buffer size based on IOContext - return new CSIndexInput(stream, entry.offset, entry.length, BufferedIndexInput.BUFFER_SIZE); + return new CSIndexInput(stream, entry.offset, entry.length, context); } /** Returns an array of strings, one for each file in the directory. */ @@ -221,13 +220,9 @@ IndexInput base; long fileOffset; long length; - - CSIndexInput(final IndexInput base, final long fileOffset, final long length) { - this(base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE); - } - - CSIndexInput(final IndexInput base, final long fileOffset, final long length, int readBufferSize) { - super(readBufferSize); + + CSIndexInput(final IndexInput base, final long fileOffset, final long length, IOContext context) { + super(context); this.base = (IndexInput)base.clone(); this.fileOffset = fileOffset; this.length = length; Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -634,7 +634,7 @@ // readBufferSize = BufferedIndexInput.BUFFER_SIZE; // } - // nocommit context should be part of the key used to cache that reader in the pool. + // TODO: context should be part of the key used to cache that reader in the pool. SegmentReader sr = readerMap.get(info); if (sr == null) { @@ -3580,8 +3580,6 @@ // keep deletes (it's costly to open entire reader // when we just need deletes) - // nocommit should we use another flag "isMergedSegment" or a "READ" context here? - final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, context, termsIndexDivisor); try { if (poolReaders && mergedSegmentWarmer != null) { Index: lucene/src/java/org/apache/lucene/index/SegmentNorms.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentNorms.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/index/SegmentNorms.java (working copy) @@ -220,7 +220,6 @@ // NOTE: norms are re-written in regular directory, not cfs si.advanceNormGen(this.number); final String normFileName = si.getNormFileName(this.number); - //nocommit not sure if this is the correct information provided to FlushInfo IndexOutput out = owner.directory().createOutput(normFileName, new IOContext(new FlushInfo(si.docCount, 0))); boolean success = false; try { Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ChecksumIndexOutput; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; @@ -57,8 +58,7 @@ @Override public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) throws IOException { - //nocommit should this context always be flush? - IndexOutput out = createOutput(dir, segmentFileName, context); + IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount()))); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT Index: lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java =================================================================== --- lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java (working copy) @@ -22,8 +22,19 @@ /** Base implementation class for buffered {@link IndexInput}. */ public abstract class BufferedIndexInput extends IndexInput { - /** Default buffer size */ + /** Default buffer size set to 1024*/ public static final int BUFFER_SIZE = 1024; + + // The normal read buffer size defaults to 1024, but + // increasing this during merging seems to yield + // performance gains. However we don't want to increase + // it too much because there are quite a few + // BufferedIndexInputs created during merging. See + // LUCENE-888 for details. + /** + * A buffer size for merges set to 4096 + */ + public static final int MERGE_BUFFER_SIZE = 4069; private int bufferSize = BUFFER_SIZE; @@ -41,11 +52,14 @@ } public BufferedIndexInput() {} + + public BufferedIndexInput(IOContext context) { + this(bufferSize(context)); + } /** Inits BufferedIndexInput with a specific bufferSize */ - //nocommit To cleanup class variable bufferSize as the the default size is always used public BufferedIndexInput(int bufferSize) { - checkBufferSize(BufferedIndexInput.BUFFER_SIZE); + checkBufferSize(bufferSize); this.bufferSize = bufferSize; } @@ -301,4 +315,21 @@ } } + /** + * Returns default buffer sizes for the given {@link IOContext} + */ + public static int bufferSize(IOContext context) { + switch (context.context) { + case DEFAULT: + case FLUSH: + case READ: + return BUFFER_SIZE; + case MERGE: + return MERGE_BUFFER_SIZE; + default: + assert false : "unknown IOContext " + context.context; + return BUFFER_SIZE; + } + } + } Index: lucene/src/java/org/apache/lucene/store/FlushInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/store/FlushInfo.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/store/FlushInfo.java (working copy) @@ -42,4 +42,36 @@ this.estimatedSegmentSize = estimatedSegmentSize; } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + (int) (estimatedSegmentSize ^ (estimatedSegmentSize >>> 32)); + result = prime * result + numDocs; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + FlushInfo other = (FlushInfo) obj; + if (estimatedSegmentSize != other.estimatedSegmentSize) + return false; + if (numDocs != other.numDocs) + return false; + return true; + } + + @Override + public String toString() { + return "FlushInfo [numDocs=" + numDocs + ", estimatedSegmentSize=" + + estimatedSegmentSize + "]"; + } + } Index: lucene/src/java/org/apache/lucene/store/IOContext.java =================================================================== --- lucene/src/java/org/apache/lucene/store/IOContext.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/store/IOContext.java (working copy) @@ -86,4 +86,47 @@ this.flushInfo = null; } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((context == null) ? 0 : context.hashCode()); + result = prime * result + ((flushInfo == null) ? 0 : flushInfo.hashCode()); + result = prime * result + ((mergeInfo == null) ? 0 : mergeInfo.hashCode()); + result = prime * result + (readOnce ? 1231 : 1237); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + IOContext other = (IOContext) obj; + if (context != other.context) + return false; + if (flushInfo == null) { + if (other.flushInfo != null) + return false; + } else if (!flushInfo.equals(other.flushInfo)) + return false; + if (mergeInfo == null) { + if (other.mergeInfo != null) + return false; + } else if (!mergeInfo.equals(other.mergeInfo)) + return false; + if (readOnce != other.readOnce) + return false; + return true; + } + + @Override + public String toString() { + return "IOContext [context=" + context + ", mergeInfo=" + mergeInfo + + ", flushInfo=" + flushInfo + ", readOnce=" + readOnce + "]"; + } + } \ No newline at end of file Index: lucene/src/java/org/apache/lucene/store/MergeInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/store/MergeInfo.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/store/MergeInfo.java (working copy) @@ -23,13 +23,13 @@ public class MergeInfo { - public int totalDocCount; + public final int totalDocCount; - public long estimatedMergeBytes; // used by IndexWriter + public final long estimatedMergeBytes; - boolean isExternal; // used by IndexWriter + public final boolean isExternal; - boolean optimize; // used by IndexWriter + public final boolean optimize; /** @@ -46,4 +46,44 @@ this.isExternal = isExternal; this.optimize = optimize; } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + (int) (estimatedMergeBytes ^ (estimatedMergeBytes >>> 32)); + result = prime * result + (isExternal ? 1231 : 1237); + result = prime * result + (optimize ? 1231 : 1237); + result = prime * result + totalDocCount; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + MergeInfo other = (MergeInfo) obj; + if (estimatedMergeBytes != other.estimatedMergeBytes) + return false; + if (isExternal != other.isExternal) + return false; + if (optimize != other.optimize) + return false; + if (totalDocCount != other.totalDocCount) + return false; + return true; + } + + @Override + public String toString() { + return "MergeInfo [totalDocCount=" + totalDocCount + + ", estimatedMergeBytes=" + estimatedMergeBytes + ", isExternal=" + + isExternal + ", optimize=" + optimize + "]"; + } } \ No newline at end of file Index: lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 1143375) +++ lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (working copy) @@ -21,7 +21,9 @@ import java.io.IOException; import java.io.RandomAccessFile; +import org.apache.lucene.store.IOContext.Context; + /** A straightforward implementation of {@link FSDirectory} * using java.io.RandomAccessFile. However, this class has * poor concurrent performance (multiple threads will @@ -87,8 +89,7 @@ protected final int chunkSize; public SimpleFSIndexInput(File path, IOContext context, int chunkSize) throws IOException { - //nocommit Use IOContext to decide bufferSize instead of BufferedIndexInput.BUFFER_SIZE - super(BufferedIndexInput.BUFFER_SIZE); + super(context); file = new Descriptor(path, "r"); this.chunkSize = chunkSize; } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java (revision 1143375) +++ lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java (working copy) @@ -124,7 +124,6 @@ indexInterval = interval; fieldInfos = fis; isIndex = isi; - // nocommit pass IOContext in via ctor output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION : PreFlexCodec.TERMS_EXTENSION)), IOContext.DEFAULT); Index: lucene/src/test/org/apache/lucene/index/TestCompoundFile.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (revision 1143375) +++ lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (working copy) @@ -321,14 +321,15 @@ private void demo_FSIndexInputBug(Directory fsdir, String file) throws IOException { + // IOContext triggers different buffer sizes so we use default here // Setup the test file - we need more than 1024 bytes - IndexOutput os = fsdir.createOutput(file, newIOContext(random)); + IndexOutput os = fsdir.createOutput(file, IOContext.DEFAULT); for(int i=0; i<2000; i++) { os.writeByte((byte) i); } os.close(); - IndexInput in = fsdir.openInput(file, newIOContext(random)); + IndexInput in = fsdir.openInput(file, IOContext.DEFAULT); // This read primes the buffer in IndexInput in.readByte();