Index: contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java =================================================================== --- contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (revision 1136453) +++ contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (working copy) @@ -19,6 +19,8 @@ import java.io.File; import java.io.IOException; + +import org.apache.lucene.index.IOContext; import org.apache.lucene.store.Directory; // javadoc import org.apache.lucene.store.NativeFSLockFactory; // javadoc @@ -67,9 +69,9 @@ } @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new WindowsIndexInput(new File(getDirectory(), name), Math.max(bufferSize, DEFAULT_BUFFERSIZE)); + return new WindowsIndexInput(new File(getDirectory(), name), DEFAULT_BUFFERSIZE); } protected static class WindowsIndexInput extends BufferedIndexInput { Index: contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java =================================================================== --- contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java (revision 1136453) +++ contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java (working copy) @@ -24,8 +24,11 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexWriter; // javadocs +import org.apache.lucene.index.MergeInfo; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.store.RAMDirectory; // javadocs @@ -193,17 +196,17 @@ } @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.createOutput name=" + name); } - if (doCacheWrite(name)) { + if (doCacheWrite(name, context)) { if (VERBOSE) { System.out.println(" to cache"); } - return cache.createOutput(name); + return cache.createOutput(name, context); } else { - return delegate.createOutput(name); + return delegate.createOutput(name, context); } } @@ -219,7 +222,7 @@ } @Override - public synchronized IndexInput openInput(String name) throws IOException { + public synchronized IndexInput openInput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.openInput name=" + name); } @@ -227,21 +230,12 @@ if (VERBOSE) { System.out.println(" from cache"); } - return cache.openInput(name); + return cache.openInput(name, context); } else { - return delegate.openInput(name); + return delegate.openInput(name, context); } } - @Override - public synchronized IndexInput openInput(String name, int bufferSize) throws IOException { - if (cache.fileExists(name)) { - return cache.openInput(name, bufferSize); - } else { - return delegate.openInput(name, bufferSize); - } - } - /** Close this directory, which flushes any cached files * to the delegate and then closes the delegate. */ @Override @@ -271,18 +265,19 @@ /** Subclass can override this to customize logic; return * true if this file should be written to the RAMDirectory. */ - protected boolean doCacheWrite(String name) { - final MergePolicy.OneMerge merge = merges.get(Thread.currentThread()); + protected boolean doCacheWrite(String name, IOContext context) { + final IOContext.MergeInfo merge = context.mergeInfo; //System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes)); return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes <= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes; } private void unCache(String fileName) throws IOException { final IndexOutput out; + IOContext context = IOContext.DEFAULT_IOCONTEXT; synchronized(this) { if (!delegate.fileExists(fileName)) { assert cache.fileExists(fileName); - out = delegate.createOutput(fileName); + out = delegate.createOutput(fileName, context); } else { out = null; } @@ -291,7 +286,7 @@ if (out != null) { IndexInput in = null; try { - in = cache.openInput(fileName); + in = cache.openInput(fileName, context); in.copyBytes(out, in.length()); } finally { IOUtils.closeSafely(false, in, out); Index: contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java =================================================================== --- contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (revision 1136453) +++ contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (working copy) @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import org.apache.lucene.index.IOContext; import org.apache.lucene.store.Directory; // javadoc import org.apache.lucene.store.NativeFSLockFactory; // javadoc @@ -69,13 +70,13 @@ } @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? bufferSize : forcedBufferSize); + return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexInput.BUFFER_SIZE : forcedBufferSize); } @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name,IOContext context) throws IOException { ensureOpen(); ensureCanWrite(name); return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexOutput.BUFFER_SIZE : forcedBufferSize); Index: src/java/org/apache/lucene/index/SegmentNorms.java =================================================================== --- src/java/org/apache/lucene/index/SegmentNorms.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentNorms.java (working copy) @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -219,7 +221,7 @@ // NOTE: norms are re-written in regular directory, not cfs si.advanceNormGen(this.number); final String normFileName = si.getNormFileName(this.number); - IndexOutput out = owner.directory().createOutput(normFileName); + IndexOutput out = owner.directory().createOutput(normFileName, IOContext.DEFAULT_IOCONTEXT); boolean success = false; try { try { Index: src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- src/java/org/apache/lucene/index/FieldInfos.java (revision 1136453) +++ src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -271,7 +271,7 @@ */ public FieldInfos(Directory d, String name) throws IOException { this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only - final IndexInput input = d.openInput(name); + final IndexInput input = d.openInput(name, new IOContext(true)); try { read(input, name); } finally { @@ -564,7 +564,7 @@ } public void write(Directory d, String name) throws IOException { - IndexOutput output = d.createOutput(name); + IndexOutput output = d.createOutput(name, new IOContext(true)); try { write(output); } finally { Index: src/java/org/apache/lucene/index/FieldsReader.java =================================================================== --- src/java/org/apache/lucene/index/FieldsReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/FieldsReader.java (working copy) @@ -25,6 +25,8 @@ import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.NumericField; +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; @@ -81,7 +83,7 @@ /** Verifies that the code version which wrote the segment is supported. */ public static void checkCodeVersion(Directory dir, String segment) throws IOException { final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); - IndexInput idxStream = dir.openInput(indexStreamFN, 1024); + IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT_IOCONTEXT); try { int format = idxStream.readInt(); @@ -110,18 +112,18 @@ } public FieldsReader(Directory d, String segment, FieldInfos fn) throws IOException { - this(d, segment, fn, BufferedIndexInput.BUFFER_SIZE, -1, 0); + this(d, segment, fn, IOContext.DEFAULT_IOCONTEXT, -1, 0); } - public FieldsReader(Directory d, String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size) throws IOException { + public FieldsReader(Directory d, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException { boolean success = false; isOriginal = true; try { fieldInfos = fn; - cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), readBufferSize); + cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); - cloneableIndexStream = d.openInput(indexStreamFN, readBufferSize); + cloneableIndexStream = d.openInput(indexStreamFN, context); format = cloneableIndexStream.readInt(); Index: src/java/org/apache/lucene/index/FieldsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FieldsWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/FieldsWriter.java (working copy) @@ -65,14 +65,14 @@ private IndexOutput fieldsStream; private IndexOutput indexStream; - FieldsWriter(Directory directory, String segment) throws IOException { + FieldsWriter(Directory directory, String segment, IOContext context) throws IOException { this.directory = directory; this.segment = segment; boolean success = false; try { - fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); - indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); + fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); + indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), context); fieldsStream.writeInt(FORMAT_CURRENT); indexStream.writeInt(FORMAT_CURRENT); Index: src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- src/java/org/apache/lucene/index/SegmentInfo.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -245,7 +245,7 @@ } final Directory dirToTest; if (isCompoundFile) { - dirToTest = new CompoundFileReader(dir, IndexFileNames.segmentFileName(storesSegment, "", ext)); + dirToTest = new CompoundFileReader(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), new IOContext(true)); } else { dirToTest = dir; } @@ -264,7 +264,7 @@ Directory dir0 = dir; if (isCompoundFile && checkCompoundFile) { dir0 = new CompoundFileReader(dir, IndexFileNames.segmentFileName(name, - "", IndexFileNames.COMPOUND_FILE_EXTENSION)); + "", IndexFileNames.COMPOUND_FILE_EXTENSION), new IOContext(true)); } try { fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name, Index: src/java/org/apache/lucene/index/values/IntsImpl.java =================================================================== --- src/java/org/apache/lucene/index/values/IntsImpl.java (revision 1136453) +++ src/java/org/apache/lucene/index/values/IntsImpl.java (working copy) @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -58,11 +59,11 @@ private int lastDocId = -1; private IndexOutput datOut; - protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed) + protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed, IOContext context) throws IOException { super(bytesUsed); datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - DATA_EXTENSION)); + DATA_EXTENSION), context); boolean success = false; try { CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT); @@ -185,9 +186,9 @@ private final IndexInput datIn; private final boolean packed; - protected IntsReader(Directory dir, String id) throws IOException { + protected IntsReader(Directory dir, String id, IOContext context) throws IOException { datIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); boolean success = false; try { CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START); Index: src/java/org/apache/lucene/index/values/Bytes.java =================================================================== --- src/java/org/apache/lucene/index/values/Bytes.java (revision 1136453) +++ src/java/org/apache/lucene/index/values/Bytes.java (working copy) @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; @@ -344,18 +345,18 @@ protected BytesWriterBase(Directory dir, String id, String codecName, int version, boolean initIndex, ByteBlockPool pool, - AtomicLong bytesUsed) throws IOException { + AtomicLong bytesUsed, IOContext context) throws IOException { super(bytesUsed); this.id = id; this.pool = pool; datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - DATA_EXTENSION)); + DATA_EXTENSION), context); boolean success = false; try { CodecUtil.writeHeader(datOut, codecName, version); if (initIndex) { idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - INDEX_EXTENSION)); + INDEX_EXTENSION), context); CodecUtil.writeHeader(idxOut, codecName, version); } else { idxOut = null; @@ -428,16 +429,16 @@ protected final String id; protected BytesReaderBase(Directory dir, String id, String codecName, - int maxVersion, boolean doIndex) throws IOException { + int maxVersion, boolean doIndex, IOContext context) throws IOException { this.id = id; datIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); boolean success = false; try { version = CodecUtil.checkHeader(datIn, codecName, maxVersion, maxVersion); if (doIndex) { idxIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.INDEX_EXTENSION)); + Writer.INDEX_EXTENSION), context); final int version2 = CodecUtil.checkHeader(idxIn, codecName, maxVersion, maxVersion); assert version == version2; Index: src/java/org/apache/lucene/index/values/Floats.java =================================================================== --- src/java/org/apache/lucene/index/values/Floats.java (revision 1136453) +++ src/java/org/apache/lucene/index/values/Floats.java (working copy) @@ -20,6 +20,8 @@ import java.util.Collection; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; @@ -65,7 +67,7 @@ public static IndexDocValues getValues(Directory dir, String id, int maxDoc) throws IOException { - return new FloatsReader(dir, id, maxDoc); + return new FloatsReader(dir, id, maxDoc, new IOContext(Context.READ)); } abstract static class FloatsWriter extends Writer { @@ -76,12 +78,12 @@ private final byte precision; protected FloatsWriter(Directory dir, String id, int precision, - AtomicLong bytesUsed) throws IOException { + AtomicLong bytesUsed, IOContext context) throws IOException { super(bytesUsed); this.id = id; this.precision = (byte) precision; datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); boolean success = false; try { CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT); @@ -146,7 +148,7 @@ protected Float4Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException { - super(dir, id, 4, bytesUsed); + super(dir, id, 4, bytesUsed, new IOContext(Context.FLUSH)); } @Override @@ -189,7 +191,7 @@ protected Float8Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException { - super(dir, id, 8, bytesUsed); + super(dir, id, 8, bytesUsed, new IOContext(Context.FLUSH)); } @Override @@ -237,10 +239,10 @@ // TODO(simonw) is ByteBuffer the way to go here? private final int maxDoc; - protected FloatsReader(Directory dir, String id, int maxDoc) + protected FloatsReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException { datIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START); precisionBytes = datIn.readByte(); assert precisionBytes == 4 || precisionBytes == 8; Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -89,8 +89,8 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor); + public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { + return get(readOnly, si.dir, si, true, termInfosIndexDivisor, context); } /** @@ -100,25 +100,25 @@ public static SegmentReader get(boolean readOnly, Directory dir, SegmentInfo si, - int readBufferSize, boolean doOpenStores, - int termInfosIndexDivisor) + int termInfosIndexDivisor, + IOContext context) throws CorruptIndexException, IOException { SegmentReader instance = new SegmentReader(); instance.readOnly = readOnly; instance.si = si; - instance.readBufferSize = readBufferSize; +// instance.readBufferSize = readBufferSize; boolean success = false; try { - instance.core = new SegmentCoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor); + instance.core = new SegmentCoreReaders(instance, dir, si, context, termInfosIndexDivisor); if (doOpenStores) { instance.core.openDocStores(si); } instance.loadDeletedDocs(); - instance.openNorms(instance.core.cfsDir, readBufferSize); + instance.openNorms(instance.core.cfsDir, context); success = true; } finally { @@ -162,9 +162,9 @@ private void loadDeletedDocs() throws IOException { // NOTE: the bitvector is stored using the regular directory, not cfs if (hasDeletions(si)) { - deletedDocs = new BitVector(directory(), si.getDelFileName()); + deletedDocs = new BitVector(directory(), si.getDelFileName(), IOContext.DEFAULT_IOCONTEXT); deletedDocsRef = new AtomicInteger(1); - assert checkDeletedCounts(); + assert checkDeletedCounts();//no commit just putting MERGE context here would lead to assert error. What MergeInfo should be proivided here? if (deletedDocs.size() != si.docCount) { throw new CorruptIndexException("document count mismatch: deleted docs count " + deletedDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name); } @@ -296,7 +296,7 @@ // If we are not cloning, then this will open anew // any norms that have changed: - clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), readBufferSize); + clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), IOContext.DEFAULT_IOCONTEXT); success = true; } finally { @@ -338,7 +338,7 @@ final String delFileName = si.getDelFileName(); boolean success = false; try { - deletedDocs.write(directory(), delFileName); + deletedDocs.write(directory(), delFileName, IOContext.DEFAULT_IOCONTEXT); success = true; } finally { if (!success) { @@ -577,7 +577,7 @@ norm.copyOnWrite()[doc] = value; // set the value } - private void openNorms(Directory cfsDir, int readBufferSize) throws IOException { + private void openNorms(Directory cfsDir, IOContext context) throws IOException { long nextNormSeek = SegmentNorms.NORMS_HEADER.length; //skip header (header unused for now) int maxDoc = maxDoc(); for (FieldInfo fi : core.fieldInfos) { @@ -601,7 +601,7 @@ if (singleNormFile) { normSeek = nextNormSeek; if (singleNormStream == null) { - singleNormStream = d.openInput(fileName, readBufferSize); + singleNormStream = d.openInput(fileName, context); singleNormRef = new AtomicInteger(1); } else { singleNormRef.incrementAndGet(); @@ -611,7 +611,7 @@ // If this were to change in the future, a clone could be done here. normInput = singleNormStream; } else { - normInput = d.openInput(fileName); + normInput = d.openInput(fileName, context); // if the segment was created in 3.2 or after, we wrote the header for sure, // and don't need to do the sketchy file size check. otherwise, we check // if the size is exactly equal to maxDoc to detect a headerless file. Index: src/java/org/apache/lucene/index/BufferedDeletesStream.java =================================================================== --- src/java/org/apache/lucene/index/BufferedDeletesStream.java (revision 1136453) +++ src/java/org/apache/lucene/index/BufferedDeletesStream.java (working copy) @@ -224,7 +224,7 @@ // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); - final SegmentReader reader = readerPool.get(info, false); + final SegmentReader reader = readerPool.get(info, false, IOContext.DEFAULT_IOCONTEXT); int delCount = 0; final boolean segAllDeletes; try { @@ -273,7 +273,7 @@ if (coalescedDeletes != null) { // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); - SegmentReader reader = readerPool.get(info, false); + SegmentReader reader = readerPool.get(info, false, IOContext.DEFAULT_IOCONTEXT); int delCount = 0; final boolean segAllDeletes; try { Index: src/java/org/apache/lucene/index/CompoundFileReader.java =================================================================== --- src/java/org/apache/lucene/index/CompoundFileReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/CompoundFileReader.java (working copy) @@ -49,21 +49,17 @@ private IndexInput stream; private HashMap entries = new HashMap(); - - public CompoundFileReader(Directory dir, String name) throws IOException { - this(dir, name, BufferedIndexInput.BUFFER_SIZE); - } - - public CompoundFileReader(Directory dir, String name, int readBufferSize) throws IOException { + + public CompoundFileReader(Directory dir, String name, IOContext context) throws IOException { assert !(dir instanceof CompoundFileReader) : "compound file inside of compound file: " + name; directory = dir; fileName = name; - this.readBufferSize = readBufferSize; +// this.readBufferSize = readBufferSize; boolean success = false; try { - stream = dir.openInput(name, readBufferSize); + stream = dir.openInput(name, context); // read the first VInt. If it is negative, it's the version number // otherwise it's the count (pre-3.1 indexes) @@ -141,13 +137,7 @@ } @Override - public synchronized IndexInput openInput(String id) throws IOException { - // Default to readBufferSize passed in when we were opened - return openInput(id, readBufferSize); - } - - @Override - public synchronized IndexInput openInput(String id, int readBufferSize) throws IOException { + public synchronized IndexInput openInput(String id, IOContext context) throws IOException { if (stream == null) throw new IOException("Stream closed"); @@ -209,7 +199,7 @@ /** Not implemented * @throws UnsupportedOperationException */ @Override - public IndexOutput createOutput(String name) { + public IndexOutput createOutput(String name, IOContext context) { throw new UnsupportedOperationException(); } Index: src/java/org/apache/lucene/index/CompoundFileWriter.java =================================================================== --- src/java/org/apache/lucene/index/CompoundFileWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/CompoundFileWriter.java (working copy) @@ -78,6 +78,7 @@ private Directory directory; private String fileName; + private IOContext context; private HashSet ids; private LinkedList entries; private boolean merged = false; @@ -87,11 +88,11 @@ * entire name (no extensions are added). * @throws NullPointerException if dir or name is null */ - public CompoundFileWriter(Directory dir, String name) { - this(dir, name, null); + public CompoundFileWriter(Directory dir, String name, IOContext context) { + this(dir, name, context, null); } - CompoundFileWriter(Directory dir, String name, MergeState.CheckAbort checkAbort) { + CompoundFileWriter(Directory dir, String name, IOContext context, MergeState.CheckAbort checkAbort) { if (dir == null) throw new NullPointerException("directory cannot be null"); if (name == null) @@ -101,6 +102,7 @@ fileName = name; ids = new HashSet(); entries = new LinkedList(); + this.context=context; } /** Returns the directory of the compound file. */ @@ -164,7 +166,7 @@ merged = true; // open the compound stream - IndexOutput os = directory.createOutput(fileName); + IndexOutput os = directory.createOutput(fileName, context); IOException priorException = null; try { // Write the Version info - must be a VInt because CFR reads a VInt @@ -228,7 +230,7 @@ * output stream. */ private void copyFile(FileEntry source, IndexOutput os) throws IOException { - IndexInput is = source.dir.openInput(source.file); + IndexInput is = source.dir.openInput(source.file, context); try { long startPtr = os.getFilePointer(); long length = is.length(); Index: src/java/org/apache/lucene/index/SegmentReadState.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReadState.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentReadState.java (working copy) @@ -26,7 +26,7 @@ public final Directory dir; public final SegmentInfo segmentInfo; public final FieldInfos fieldInfos; - public final int readBufferSize; + public final IOContext context; // NOTE: if this is < 0, that means "defer terms index // load until needed". But if the codec must load the @@ -37,20 +37,20 @@ public final int codecId; public SegmentReadState(Directory dir, SegmentInfo info, - FieldInfos fieldInfos, int readBufferSize, int termsIndexDivisor) { - this(dir, info, fieldInfos, readBufferSize, termsIndexDivisor, -1); + FieldInfos fieldInfos, IOContext context, int termsIndexDivisor) { + this(dir, info, fieldInfos, context, termsIndexDivisor, -1); } public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, - int readBufferSize, + IOContext context, int termsIndexDivisor, int codecId) { this.dir = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; - this.readBufferSize = readBufferSize; + this.context = context; this.termsIndexDivisor = termsIndexDivisor; this.codecId = codecId; } Index: src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- src/java/org/apache/lucene/index/SegmentInfos.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -32,6 +32,7 @@ import java.util.Set; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.SegmentInfosReader; @@ -254,7 +255,7 @@ try { SegmentInfosReader infosReader = codecs.getSegmentInfosReader(); - infosReader.read(directory, segmentFileName, codecs, this); + infosReader.read(directory, segmentFileName, codecs, this, new IOContext(Context.READ)); success = true; } finally { @@ -322,7 +323,7 @@ try { SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); - segnOutput = infosWriter.writeInfos(directory, segmentFileName, this); + segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT_IOCONTEXT); infosWriter.prepareCommit(segnOutput); pendingSegnOutput = segnOutput; success = true; @@ -597,7 +598,7 @@ for(int i=0;i= 0; i--) { boolean success = false; try { - readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor); + readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor, new IOContext(Context.READ)); readers[i].readerFinishedListeners = readerFinishedListeners; success = true; } finally { @@ -170,7 +171,8 @@ try { final SegmentInfo info = infos.info(i); assert info.dir == dir; - final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor); + final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor, + new IOContext(Context.READ)); if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) { reader.readerFinishedListeners = readerFinishedListeners; readers.add(reader); @@ -254,7 +256,7 @@ assert !doClone; // this is a new reader; in case we hit an exception we can close it safely - newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor); + newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor, new IOContext(Context.READ)); newReader.readerFinishedListeners = readerFinishedListeners; } else { newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly); Index: src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- src/java/org/apache/lucene/index/SegmentMerger.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.document.Document; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexReader.FieldOption; import org.apache.lucene.index.MergePolicy.MergeAbortedException; import org.apache.lucene.index.codecs.Codec; @@ -132,7 +133,8 @@ // Now merge all added files Collection files = info.files(); - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort); + //no commit just putting MERGE context here would lead to assert error. What MergeInfo should be proivided here? + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, new IOContext(Context.MERGE), checkAbort); for (String file : files) { assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION) : ".del file is not allowed in .cfs: " + file; @@ -233,7 +235,7 @@ setMatchingSegmentReaders(); - final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment); + final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, IOContext.DEFAULT_IOCONTEXT); try { int idx = 0; @@ -269,7 +271,7 @@ // details. throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null); + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null, new IOContext(Context.MERGE,new IOContext.MergeInfo(docCount, 0, false, false))); return docCount; } @@ -354,8 +356,9 @@ * @throws IOException */ private final void mergeVectors() throws IOException { + //no commit just putting MERGE context here would lead to assert error. What MergeInfo should be proivided here? TermVectorsWriter termVectorsWriter = - new TermVectorsWriter(directory, segment, fieldInfos); + new TermVectorsWriter(directory, segment, fieldInfos, new IOContext(Context.MERGE)); try { int idx = 0; @@ -614,7 +617,8 @@ for (FieldInfo fi : fieldInfos) { if (fi.isIndexed && !fi.omitNorms) { if (output == null) { - output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION)); + //no commit just putting MERGE context here would lead to assert error. What MergeInfo should be proivided here? + output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION), new IOContext(Context.MERGE)); output.writeBytes(SegmentNorms.NORMS_HEADER, SegmentNorms.NORMS_HEADER.length); } for (IndexReader reader : readers) { Index: src/java/org/apache/lucene/index/IOContext.java =================================================================== --- src/java/org/apache/lucene/index/IOContext.java (revision 0) +++ src/java/org/apache/lucene/index/IOContext.java (revision 0) @@ -0,0 +1,95 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * +* Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +IOContext holds additional details on the merge/search context. +A IOContext object can never be initialised as null as passed as a parameter to either +{@link #org.apache.lucene.store.Directory.openInput()} or {@link #org.apache.lucene.store.Directory.createInput()} + */ +public class IOContext { + + /** + * Context is a enumerator which specifies the context in which the Directory is being used for. + */ + public enum Context {MERGE,READ,FLUSH,DEFAULT}; + + /** + * An object of a enumerator Context type + */ + public final Context context; + + /** + * An object of {@link #org.apache.lucene.index.MergePolicy.OneMerge} + */ +// public final OneMerge onemerge; +// public final MergeInfo mergeInfo; + public final MergeInfo mergeInfo; + + public final boolean readOnce; + + public static final IOContext DEFAULT_IOCONTEXT = new IOContext(Context.DEFAULT); + + public IOContext () { + this(false); + } + + public IOContext(Context context) { + this.context = context; + this.mergeInfo = null; + this.readOnce = false; + assert context == Context.MERGE; + } + + public IOContext(boolean readOnce) { + this.context = Context.DEFAULT; + this.mergeInfo = null; + this.readOnce = readOnce; + } + + public IOContext (MergePolicy.OneMerge oneMerge) { + this.context = Context.MERGE; + this.readOnce = false; + mergeInfo = new MergeInfo(oneMerge.totalDocCount, oneMerge.estimatedMergeBytes , oneMerge.isExternal , oneMerge.optimize); + } + + public IOContext (Context context, MergeInfo mergeinfo ) { + this.context = context; + this.readOnce = false; + this.mergeInfo = mergeinfo; + } + + public static class MergeInfo { + + public int totalDocCount; + + public long estimatedMergeBytes; + + boolean isExternal; // used by IndexWriter + + boolean optimize; // used by IndexWriter + + MergeInfo(int totalDocCount,long estimatedMergeBytes, boolean isExternal, boolean optimize) { + this.totalDocCount = totalDocCount; + this.estimatedMergeBytes = estimatedMergeBytes; + this.isExternal = isExternal; + this.optimize = optimize; + } + } + +} \ No newline at end of file Index: src/java/org/apache/lucene/index/TermVectorsTermsWriter.java =================================================================== --- src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Map; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -89,13 +90,14 @@ if (tvx == null) { boolean success = false; try { + IOContext context = new IOContext(Context.FLUSH); // If we hit an exception while init'ing the term // vector output files, we must abort this segment // because those files will be in an unknown // state: - tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION)); - tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)); - tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION)); + tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION), context); + tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), context); + tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION), context); tvx.writeInt(TermVectorsReader.FORMAT_CURRENT); tvd.writeInt(TermVectorsReader.FORMAT_CURRENT); Index: src/java/org/apache/lucene/index/PerFieldCodecWrapper.java =================================================================== --- src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (revision 1136453) +++ src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (working copy) @@ -99,7 +99,7 @@ private final Map codecs = new HashMap(); public FieldsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo si, - int readBufferSize, int indexDivisor) throws IOException { + IOContext context, int indexDivisor) throws IOException { final Map producers = new HashMap(); boolean success = false; @@ -111,7 +111,7 @@ Codec codec = segmentCodecs.codecs[fi.getCodecId()]; if (!producers.containsKey(codec)) { producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir, - si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId()))); + si, fieldInfos, context, indexDivisor, fi.getCodecId()))); } codecs.put(fi.name, producers.get(codec)); } @@ -187,7 +187,7 @@ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { return new FieldsReader(state.dir, state.fieldInfos, state.segmentInfo, - state.readBufferSize, state.termsIndexDivisor); + state.context, state.termsIndexDivisor); } @Override @@ -212,14 +212,14 @@ @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { return new PerDocProducers(state.dir, state.fieldInfos, state.segmentInfo, - state.readBufferSize, state.termsIndexDivisor); + state.context, state.termsIndexDivisor); } private final class PerDocProducers extends PerDocValues { private final TreeMap codecs = new TreeMap(); public PerDocProducers(Directory dir, FieldInfos fieldInfos, SegmentInfo si, - int readBufferSize, int indexDivisor) throws IOException { + IOContext context, int indexDivisor) throws IOException { final Map producers = new HashMap(); boolean success = false; try { @@ -229,7 +229,7 @@ Codec codec = segmentCodecs.codecs[fi.getCodecId()]; if (!producers.containsKey(codec)) { producers.put(codec, codec.docsProducer(new SegmentReadState(dir, - si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId()))); + si, fieldInfos, context, indexDivisor, fi.getCodecId()))); } codecs.put(fi.name, producers.get(codec)); } Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1433,13 +1433,15 @@ Directory dir = null; CompoundFileReader cfr = null; + IOContext context=null; + //TODO: Fill in IOContext try { File file = new File(filename); String dirname = file.getAbsoluteFile().getParent(); filename = file.getName(); dir = FSDirectory.open(new File(dirname)); - cfr = new CompoundFileReader(dir, filename); + cfr = new CompoundFileReader(dir, filename, context); String [] files = cfr.listAll(); ArrayUtil.mergeSort(files); // sort the array of filename so that the output is more readable @@ -1449,7 +1451,7 @@ if (extract) { System.out.println("extract " + files[i] + " with " + len + " bytes to local directory..."); - IndexInput ii = cfr.openInput(files[i]); + IndexInput ii = cfr.openInput(files[i], context); FileOutputStream f = new FileOutputStream(files[i]); Index: src/java/org/apache/lucene/index/TermVectorsReader.java =================================================================== --- src/java/org/apache/lucene/index/TermVectorsReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/TermVectorsReader.java (working copy) @@ -17,6 +17,8 @@ * limitations under the License. */ +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -61,29 +63,24 @@ private final int format; - TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos) + TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context) throws CorruptIndexException, IOException { - this(d, segment, fieldInfos, BufferedIndexInput.BUFFER_SIZE); + this(d, segment, fieldInfos, context, -1, 0); } - - TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, int readBufferSize) - throws CorruptIndexException, IOException { - this(d, segment, fieldInfos, readBufferSize, -1, 0); - } - TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, int readBufferSize, int docStoreOffset, int size) + TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context, int docStoreOffset, int size) throws CorruptIndexException, IOException { boolean success = false; try { String idxName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION); - tvx = d.openInput(idxName, readBufferSize); + tvx = d.openInput(idxName, context); format = checkValidFormat(tvx, idxName); String fn = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); - tvd = d.openInput(fn, readBufferSize); + tvd = d.openInput(fn, context); final int tvdFormat = checkValidFormat(tvd, fn); fn = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); - tvf = d.openInput(fn, readBufferSize); + tvf = d.openInput(fn, context); final int tvfFormat = checkValidFormat(tvf, fn); assert format == tvdFormat; Index: src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/IndexWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -39,7 +39,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; import org.apache.lucene.index.codecs.CodecProvider; @@ -592,8 +594,8 @@ * enrolled in the pool, so you should simply close() * it when you're done (ie, do not call release()). */ - public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor) throws IOException { - SegmentReader sr = get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, termInfosIndexDivisor); + public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor, IOContext context) throws IOException { + SegmentReader sr = get(info, doOpenStores, context, termInfosIndexDivisor); try { return (SegmentReader) sr.clone(true); } finally { @@ -609,8 +611,8 @@ * @param doOpenStores * @throws IOException */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores) throws IOException { - return get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, config.getReaderTermsIndexDivisor()); + public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, IOContext context) throws IOException { + return get(info, doOpenStores, context, config.getReaderTermsIndexDivisor()); } /** @@ -624,18 +626,20 @@ * @param termsIndexDivisor * @throws IOException */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, int readBufferSize, int termsIndexDivisor) throws IOException { + public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, IOContext context, int termsIndexDivisor) throws IOException { - if (poolReaders) { - readBufferSize = BufferedIndexInput.BUFFER_SIZE; - } + // if (poolReaders) { + // readBufferSize = BufferedIndexInput.BUFFER_SIZE; + // } + + // nocommit context should be part of the key used to cache that reader in the pool. SegmentReader sr = readerMap.get(info); if (sr == null) { // TODO: we may want to avoid doing this while // synchronized // Returns a ref, which we xfer to readerMap: - sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor); + sr = SegmentReader.get(false, info.dir, info, doOpenStores, termsIndexDivisor, context); sr.readerFinishedListeners = readerFinishedListeners; if (info.dir == directory) { @@ -2180,6 +2184,8 @@ SegmentInfo newSegment = flushedSegment.segmentInfo; setDiagnostics(newSegment, "flush"); + + IOContext context = new IOContext(Context.FLUSH); boolean success = false; try { @@ -2187,7 +2193,7 @@ String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); message("creating compound file " + compoundFileName); // Now build compound file - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName, context); for(String fileName : newSegment.files()) { cfsWriter.addFile(fileName); } @@ -2219,7 +2225,7 @@ // shortly-to-be-opened SegmentReader and let it // carry the changes; there's no reason to use // filesystem as intermediary here. - flushedSegment.deletedDocuments.write(directory, delFileName); + flushedSegment.deletedDocuments.write(directory, delFileName, context); success2 = true; } finally { if (!success2) { @@ -2388,11 +2394,13 @@ // optimize case only for segments that don't share doc stores && versionComparator.compare(info.getVersion(), "3.1") >= 0; } - + IOContext.MergeInfo mergeinfo = new IOContext.MergeInfo(docCount, 0, true, false); + IOContext context=new IOContext(Context.MERGE, mergeinfo); + if (createCFS) { - copySegmentIntoCFS(info, newSegName); + copySegmentIntoCFS(info, newSegName, context); } else { - copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied); + copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context); } infos.add(info); @@ -2496,10 +2504,10 @@ } /** Copies the segment into the IndexWriter's directory, as a compound segment. */ - private void copySegmentIntoCFS(SegmentInfo info, String segName) throws IOException { + private void copySegmentIntoCFS(SegmentInfo info, String segName, IOContext context) throws IOException { String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION); Collection files = info.files(); - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segFileName); + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segFileName, context); for (String file : files) { String newFileName = segName + IndexFileNames.stripSegmentName(file); if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION) @@ -2507,7 +2515,7 @@ cfsWriter.addFile(file, info.dir); } else { assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists"; - info.dir.copy(directory, file, newFileName); + info.dir.copy(directory, file, newFileName, context); } } @@ -2521,7 +2529,7 @@ /** Copies the segment files as-is into the IndexWriter's directory. */ private void copySegmentAsIs(SegmentInfo info, String segName, - Map dsNames, Set dsFilesCopied) + Map dsNames, Set dsFilesCopied, IOContext context) throws IOException { // Determine if the doc store of this segment needs to be copied. It's // only relevant for segments that share doc store with others, @@ -2557,7 +2565,7 @@ } assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists"; - info.dir.copy(directory, file, newFileName); + info.dir.copy(directory, file, newFileName, context); } info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile()); @@ -3411,6 +3419,8 @@ int mergedDocCount = 0; List sourceSegments = merge.segments; + + IOContext context = new IOContext(Context.MERGE); SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge, payloadProcessorProvider, merge.info.getFieldInfos()); @@ -3434,7 +3444,7 @@ // Hold onto the "live" reader; we will use this to // commit merged deletes final SegmentReader reader = readerPool.get(info, true, - MERGE_READ_BUFFER_SIZE, + context, -config.getReaderTermsIndexDivisor()); merge.readers.add(reader); @@ -3558,7 +3568,9 @@ // keep deletes (it's costly to open entire reader // when we just need deletes) - final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, BufferedIndexInput.BUFFER_SIZE, termsIndexDivisor); + // no commit : should we use another flag "isMergedSegment" or a "READ" context here? + + final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, context, termsIndexDivisor); try { if (poolReaders && mergedSegmentWarmer != null) { mergedSegmentWarmer.warm(mergedReader); Index: src/java/org/apache/lucene/index/TermVectorsWriter.java =================================================================== --- src/java/org/apache/lucene/index/TermVectorsWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/TermVectorsWriter.java (working copy) @@ -31,15 +31,15 @@ private FieldInfos fieldInfos; public TermVectorsWriter(Directory directory, String segment, - FieldInfos fieldInfos) throws IOException { + FieldInfos fieldInfos, IOContext context) throws IOException { boolean success = false; try { // Open files for TermVector storage - tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION)); + tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION), context); tvx.writeInt(TermVectorsReader.FORMAT_CURRENT); - tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)); + tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), context); tvd.writeInt(TermVectorsReader.FORMAT_CURRENT); - tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION)); + tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION), context); tvf.writeInt(TermVectorsReader.FORMAT_CURRENT); success = true; } finally { Index: src/java/org/apache/lucene/index/SegmentWriteState.java =================================================================== --- src/java/org/apache/lucene/index/SegmentWriteState.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentWriteState.java (working copy) @@ -51,9 +51,11 @@ * slower. Searching is typically not dominated by dictionary lookup, so * tweaking this is rarely useful.*/ public int termIndexInterval; // TODO: this should be private to the codec, not settable here or in IWC + + public final IOContext context; public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, - int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes) { + int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes, IOContext context) { this.infoStream = infoStream; this.segDeletes = segDeletes; this.directory = directory; @@ -63,6 +65,7 @@ this.termIndexInterval = termIndexInterval; this.segmentCodecs = segmentCodecs; codecId = -1; + this.context=context; } /** @@ -76,6 +79,7 @@ numDocs = state.numDocs; termIndexInterval = state.termIndexInterval; segmentCodecs = state.segmentCodecs; + context=state.context; this.codecId = codecId; segDeletes = state.segDeletes; } Index: src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java =================================================================== --- src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (working copy) @@ -110,7 +110,7 @@ // We wrap StandardPostingsReader, but any StandardPostingsReader // will work: - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); PostingsReaderBase pulsingReader = new PulsingPostingsReaderImpl(docsReader); // Terms dict index reader @@ -136,7 +136,7 @@ FieldsProducer ret = new BlockTermsReader(indexReader, state.dir, state.fieldInfos, state.segmentInfo.name, pulsingReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; Index: src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java (working copy) @@ -72,7 +72,7 @@ throws IOException { final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; - out = state.directory.createOutput(termsFileName); + out = state.directory.createOutput(termsFileName, state.context); boolean success = false; try { fieldInfos = state.fieldInfos; Index: src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (working copy) @@ -19,8 +19,10 @@ import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.ChecksumIndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -54,9 +56,9 @@ public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS; @Override - public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos) + public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) throws IOException { - IndexOutput out = createOutput(dir, segmentFileName); + IndexOutput out = createOutput(dir, segmentFileName, context); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT @@ -77,9 +79,9 @@ } } - protected IndexOutput createOutput(Directory dir, String segmentFileName) + protected IndexOutput createOutput(Directory dir, String segmentFileName, IOContext context) throws IOException { - IndexOutput plainOut = dir.createOutput(segmentFileName); + IndexOutput plainOut = dir.createOutput(segmentFileName, context); ChecksumIndexOutput out = new ChecksumIndexOutput(plainOut); return out; } Index: src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (working copy) @@ -21,6 +21,8 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; @@ -68,12 +70,12 @@ // start of the field info data protected long dirOffset; - public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId) + public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) throws IOException { this.termComp = termComp; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); boolean success = false; Index: src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java (working copy) @@ -46,7 +46,7 @@ public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException { final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId); - out = state.directory.createOutput(fileName); + out = state.directory.createOutput(fileName, state.context); } private void write(String s) throws IOException { Index: src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -58,7 +58,7 @@ final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { - in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId)); + in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId), state.context); fieldInfos = state.fieldInfos; } Index: src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (working copy) @@ -92,14 +92,14 @@ this.skipMinimum = skipInterval; /* set to the same for now */ //this.segment = state.segmentName; String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION); - freqOut = state.directory.createOutput(fileName); + freqOut = state.directory.createOutput(fileName, state.context); boolean success = false; try { if (state.fieldInfos.hasProx()) { // At least one field does not omit TF, so create the // prox file fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); - proxOut = state.directory.createOutput(fileName); + proxOut = state.directory.createOutput(fileName, state.context); } else { // Every field omits TF so we will write no prox file proxOut = null; Index: src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; @@ -51,15 +52,15 @@ //private String segment; - public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, int codecId) throws IOException { + public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, int codecId) throws IOException { freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION), - readBufferSize); + context); //this.segment = segmentInfo.name; if (segmentInfo.getHasProx()) { boolean success = false; try { proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION), - readBufferSize); + context); success = true; } finally { if (!success) { Index: src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java =================================================================== --- src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (working copy) @@ -89,7 +89,7 @@ @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); + PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -113,7 +113,7 @@ state.fieldInfos, state.segmentInfo.name, postings, - state.readBufferSize, + state.context, TERMS_CACHE_SIZE, state.codecId); success = true; Index: src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.Directory; @@ -36,5 +37,5 @@ * @param infos empty instance to be populated with data * @throws IOException */ - public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos) throws IOException; + public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos, IOContext context) throws IOException; } Index: src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java =================================================================== --- src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (working copy) @@ -65,7 +65,7 @@ @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.readBufferSize, state.termsIndexDivisor); + return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor); } @Override Index: src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.Directory; @@ -94,7 +95,7 @@ SegmentTermEnum termEnum; } - TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize, int indexDivisor) + TermInfosReader(Directory dir, String seg, FieldInfos fis, IOContext context, int indexDivisor) throws CorruptIndexException, IOException { boolean success = false; @@ -108,7 +109,7 @@ fieldInfos = fis; origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), - readBufferSize), fieldInfos, false); + context), fieldInfos, false); size = origEnum.size; @@ -116,7 +117,7 @@ // Load terms index totalIndexInterval = origEnum.indexInterval * indexDivisor; final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION), - readBufferSize), fieldInfos, true); + context), fieldInfos, true); try { int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index Index: src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java =================================================================== --- src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.Term; @@ -62,10 +63,10 @@ final TreeMap fields = new TreeMap(); final Map preTerms = new HashMap(); private final Directory dir; - private final int readBufferSize; + private final IOContext context; private Directory cfsReader; - public PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor) + public PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) throws IOException { si = info; @@ -80,19 +81,19 @@ boolean success = false; try { - TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor); + TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, context, indexDivisor); if (indexDivisor == -1) { tisNoIndex = r; } else { tisNoIndex = null; tis = r; } - this.readBufferSize = readBufferSize; + this.context = context; this.fieldInfos = fieldInfos; // make sure that all index files have been read or are kept open // so that if an index update removes them we'll still have them - freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION), readBufferSize); + freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION), context); boolean anyProx = false; for (FieldInfo fi : fieldInfos) { if (fi.isIndexed) { @@ -105,7 +106,7 @@ } if (anyProx) { - proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION), readBufferSize); + proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION), context); } else { proxStream = null; } @@ -178,7 +179,7 @@ // to CFS if (!(dir instanceof CompoundFileReader)) { - dir0 = cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + dir0 = cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); } else { dir0 = dir; } @@ -187,7 +188,7 @@ dir0 = dir; } - tis = new TermInfosReader(dir0, si.name, fieldInfos, readBufferSize, indexDivisor); + tis = new TermInfosReader(dir0, si.name, fieldInfos, context, indexDivisor); } } Index: src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java (working copy) @@ -58,7 +58,7 @@ public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException { final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); termIndexInterval = state.termIndexInterval; - out = state.directory.createOutput(indexFileName); + out = state.directory.createOutput(indexFileName, state.context); boolean success = false; try { fieldInfos = state.fieldInfos; Index: src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -41,7 +42,7 @@ * phase commit" operations as described above. * @throws IOException */ - public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos) throws IOException; + public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException; /** * First phase of the two-phase commit - ensure that all output can be Index: src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java (working copy) @@ -27,6 +27,8 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; @@ -57,10 +59,10 @@ protected long dirOffset; final String segment; - public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId) + public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId, IOContext context) throws IOException { - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); this.segment = segment; boolean success = false; Index: src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; @@ -58,19 +59,19 @@ int maxSkipLevels; int skipMinimum; - public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory, int codecId) throws IOException { + public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, int codecId) throws IOException { boolean success = false; try { final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.DOC_EXTENSION); docIn = intFactory.openInput(dir, docFileName); - skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION), readBufferSize); + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION), context); if (segmentInfo.getHasProx()) { freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.FREQ_EXTENSION)); - posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION), readBufferSize); - payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION), readBufferSize); + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION)); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION), context); } else { posIn = null; payloadIn = null; Index: src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java (working copy) @@ -132,11 +132,11 @@ // TODO: -- only if at least one field stores payloads? final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION); - payloadOut = state.directory.createOutput(payloadFileName); + payloadOut = state.directory.createOutput(payloadFileName, state.context); } final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION); - skipOut = state.directory.createOutput(skipFileName); + skipOut = state.directory.createOutput(skipFileName, state.context); totalNumDocs = state.numDocs; Index: src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java =================================================================== --- src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import org.apache.lucene.index.IOContext; import org.apache.lucene.store.Directory; import org.apache.lucene.store.BufferedIndexInput; @@ -24,10 +25,10 @@ /** @lucene.experimental */ public abstract class IntStreamFactory { - public IntIndexInput openInput(Directory dir, String fileName) throws IOException { - return openInput(dir, fileName, BufferedIndexInput.BUFFER_SIZE); + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return openInput(dir, fileName, context); } - public abstract IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException; +// public abstract IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException; public abstract IntIndexOutput createOutput(Directory dir, String fileName) throws IOException; } Index: src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java (working copy) @@ -159,7 +159,7 @@ public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy) throws IOException { final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); - out = state.directory.createOutput(indexFileName); + out = state.directory.createOutput(indexFileName, state.context); boolean success = false; try { fieldInfos = state.fieldInfos; Index: src/java/org/apache/lucene/index/codecs/BlockTermsReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; @@ -107,7 +108,7 @@ //private String segment; - public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize, + public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, int termsCacheSize, int codecId) throws IOException { @@ -116,7 +117,7 @@ //this.segment = segment; in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTermsWriter.TERMS_EXTENSION), - readBufferSize); + context); boolean success = false; try { Index: src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (revision 1136453) +++ src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (working copy) @@ -22,11 +22,13 @@ import org.apache.lucene.index.CompoundFileReader; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldsReader; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -39,10 +41,10 @@ @Override public void read(Directory directory, String segmentsFileName, CodecProvider codecs, - SegmentInfos infos) throws IOException { + SegmentInfos infos, IOContext context) throws IOException { IndexInput input = null; try { - input = openInput(directory, segmentsFileName); + input = openInput(directory, segmentsFileName, context); final int format = input.readInt(); infos.setFormat(format); @@ -70,11 +72,11 @@ if (si.getDocStoreIsCompoundFile()) { dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( si.getDocStoreSegment(), "", - IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024); + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context); } } else if (si.getUseCompoundFile()) { dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( - si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), 1024); + si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); } try { @@ -108,8 +110,8 @@ } - public IndexInput openInput(Directory dir, String segmentsFileName) throws IOException { - IndexInput in = dir.openInput(segmentsFileName); + public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context) throws IOException { + IndexInput in = dir.openInput(segmentsFileName, context); return new ChecksumIndexInput(in); } Index: src/java/org/apache/lucene/index/SegmentCoreReaders.java =================================================================== --- src/java/org/apache/lucene/index/SegmentCoreReaders.java (revision 1136453) +++ src/java/org/apache/lucene/index/SegmentCoreReaders.java (working copy) @@ -45,7 +45,7 @@ final Directory dir; final Directory cfsDir; - final int readBufferSize; + final IOContext context; final int termsIndexDivisor; private final SegmentReader owner; @@ -57,7 +57,7 @@ - SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException { + SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, IOContext context, int termsIndexDivisor) throws IOException { if (termsIndexDivisor == 0) { throw new IllegalArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)"); @@ -65,7 +65,7 @@ segment = si.name; final SegmentCodecs segmentCodecs = si.getSegmentCodecs(); - this.readBufferSize = readBufferSize; + this.context = context; this.dir = dir; boolean success = false; @@ -73,7 +73,7 @@ try { Directory dir0 = dir; if (si.getUseCompoundFile()) { - cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); dir0 = cfsReader; } cfsDir = dir0; @@ -82,7 +82,7 @@ this.termsIndexDivisor = termsIndexDivisor; final Codec codec = segmentCodecs.codec(); - final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor); + final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor); // Ask codec for its Fields fields = codec.fieldsProducer(segmentReadState); assert fields != null; @@ -163,7 +163,7 @@ assert storeCFSReader == null; storeCFSReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), - readBufferSize); + context); storeDir = storeCFSReader; assert storeDir != null; } else { @@ -175,7 +175,7 @@ // was not used, but then we are asked to open doc // stores after the segment has switched to CFS if (cfsReader == null) { - cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); } storeDir = cfsReader; assert storeDir != null; @@ -185,7 +185,7 @@ } final String storesSegment = si.getDocStoreSegment(); - fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, readBufferSize, + fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); // Verify two sources of "maxDoc" agree: @@ -194,7 +194,7 @@ } if (si.getHasVectors()) { // open term vector files only as needed - termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.getDocStoreOffset(), si.docCount); + termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); } } } Index: src/java/org/apache/lucene/index/NormsWriter.java =================================================================== --- src/java/org/apache/lucene/index/NormsWriter.java (revision 1136453) +++ src/java/org/apache/lucene/index/NormsWriter.java (working copy) @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Map; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; @@ -49,7 +50,7 @@ } final String normsFileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.NORMS_EXTENSION); - IndexOutput normsOut = state.directory.createOutput(normsFileName); + IndexOutput normsOut = state.directory.createOutput(normsFileName, state.context); boolean success = false; try { normsOut.writeBytes(SegmentNorms.NORMS_HEADER, 0, SegmentNorms.NORMS_HEADER.length); Index: src/java/org/apache/lucene/store/NIOFSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 1136453) +++ src/java/org/apache/lucene/store/NIOFSDirectory.java (working copy) @@ -23,6 +23,7 @@ import java.nio.channels.ClosedChannelException; // javadoc @link import java.nio.channels.FileChannel; import java.util.concurrent.Future; // javadoc +import org.apache.lucene.index.IOContext; /** * An {@link FSDirectory} implementation that uses java.nio's FileChannel's @@ -73,9 +74,9 @@ /** Creates an IndexInput for the file with the given name. */ @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize()); + return new NIOFSIndexInput(new File(getDirectory(), name), context, getReadChunkSize()); } protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput { @@ -87,8 +88,8 @@ final FileChannel channel; - public NIOFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException { - super(path, bufferSize, chunkSize); + public NIOFSIndexInput(File path, IOContext context, int chunkSize) throws IOException { + super(path, context, chunkSize); channel = file.getChannel(); } @@ -178,4 +179,5 @@ } } } + } Index: src/java/org/apache/lucene/store/MMapDirectory.java =================================================================== --- src/java/org/apache/lucene/store/MMapDirectory.java (revision 1136453) +++ src/java/org/apache/lucene/store/MMapDirectory.java (working copy) @@ -31,6 +31,7 @@ import java.security.PrivilegedActionException; import java.lang.reflect.Method; +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.Constants; /** File-based {@link Directory} implementation that uses @@ -208,7 +209,7 @@ /** Creates an IndexInput for the file with the given name. */ @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); File f = new File(getDirectory(), name); RandomAccessFile raf = new RandomAccessFile(f, "r"); @@ -399,4 +400,5 @@ } } } + } Index: src/java/org/apache/lucene/store/RAMDirectory.java =================================================================== --- src/java/org/apache/lucene/store/RAMDirectory.java (revision 1136453) +++ src/java/org/apache/lucene/store/RAMDirectory.java (working copy) @@ -27,6 +27,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; + /** * A memory-resident {@link Directory} implementation. Locking * implementation is by default the {@link SingleInstanceLockFactory} @@ -38,7 +40,7 @@ // ***** // Lock acquisition sequence: RAMDirectory, then RAMFile - // ***** + // ***** /** Constructs an empty {@link Directory}. */ public RAMDirectory() { @@ -65,14 +67,14 @@ * @param dir a Directory value * @exception IOException if an error occurs */ - public RAMDirectory(Directory dir) throws IOException { - this(dir, false); + public RAMDirectory(Directory dir, IOContext context) throws IOException { + this(dir, false, context); } - private RAMDirectory(Directory dir, boolean closeDir) throws IOException { + private RAMDirectory(Directory dir, boolean closeDir, IOContext context) throws IOException { this(); for (String file : dir.listAll()) { - dir.copy(this, file, file); + dir.copy(this, file, file, context); } if (closeDir) { dir.close(); @@ -149,7 +151,7 @@ /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */ @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); RAMFile file = newRAMFile(); RAMFile existing = fileMap.remove(name); @@ -176,7 +178,7 @@ /** Returns a stream reading an existing file. */ @Override - public IndexInput openInput(String name) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); RAMFile file = fileMap.get(name); if (file == null) { Index: src/java/org/apache/lucene/store/Directory.java =================================================================== --- src/java/org/apache/lucene/store/Directory.java (revision 1136453) +++ src/java/org/apache/lucene/store/Directory.java (working copy) @@ -22,6 +22,7 @@ import java.io.Closeable; import java.util.Collection; // for javadocs +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.IOUtils; /** A Directory is a flat list of files. Files may be written once, when they @@ -87,7 +88,7 @@ /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */ - public abstract IndexOutput createOutput(String name) + public abstract IndexOutput createOutput(String name, IOContext context) throws IOException; /** @@ -103,10 +104,6 @@ */ public abstract void sync(Collection names) throws IOException; - /** Returns a stream reading an existing file. */ - public abstract IndexInput openInput(String name) - throws IOException; - /** Returns a stream reading an existing file, with the * specified read buffer size. The particular Directory * implementation may ignore the buffer size. Currently @@ -114,8 +111,8 @@ * parameter are {@link FSDirectory} and {@link * org.apache.lucene.index.CompoundFileReader}. */ - public IndexInput openInput(String name, int bufferSize) throws IOException { - return openInput(name); + public IndexInput openInput(String name, IOContext context) throws IOException { + return openInput(name, context); } /** Construct a {@link Lock}. @@ -199,9 +196,9 @@ * NOTE: this method does not check whether dest exist and will * overwrite it if it does. */ - public void copy(Directory to, String src, String dest) throws IOException { - IndexOutput os = to.createOutput(dest); - IndexInput is = openInput(src); + public void copy(Directory to, String src, String dest, IOContext context) throws IOException { + IndexOutput os = to.createOutput(dest, context); + IndexInput is = openInput(src, context); IOException priorException = null; try { is.copyBytes(os, is.length()); Index: src/java/org/apache/lucene/store/SimpleFSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 1136453) +++ src/java/org/apache/lucene/store/SimpleFSDirectory.java (working copy) @@ -21,6 +21,8 @@ import java.io.IOException; import java.io.RandomAccessFile; +import org.apache.lucene.index.IOContext; + /** A straightforward implementation of {@link FSDirectory} * using java.io.RandomAccessFile. However, this class has * poor concurrent performance (multiple threads will @@ -51,9 +53,9 @@ /** Creates an IndexInput for the file with the given name. */ @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize()); + return new SimpleFSIndexInput(new File(directory, name), context, getReadChunkSize()); } protected static class SimpleFSIndexInput extends BufferedIndexInput { @@ -85,8 +87,9 @@ // LUCENE-1566 - maximum read length on a 32bit JVM to prevent incorrect OOM protected final int chunkSize; - public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException { - super(bufferSize); + public SimpleFSIndexInput(File path, IOContext context, int chunkSize) throws IOException { + //TODO Use IOContext to decide bufferSize instead of BufferedIndexInput.BUFFER_SIZE + super(BufferedIndexInput.BUFFER_SIZE); file = new Descriptor(path, "r"); this.chunkSize = chunkSize; } Index: src/java/org/apache/lucene/store/FileSwitchDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FileSwitchDirectory.java (revision 1136453) +++ src/java/org/apache/lucene/store/FileSwitchDirectory.java (working copy) @@ -25,6 +25,8 @@ import java.util.Set; import java.util.HashSet; +import org.apache.lucene.index.IOContext; + /** * Expert: A Directory instance that switches files between * two other Directory instances. @@ -125,8 +127,8 @@ } @Override - public IndexOutput createOutput(String name) throws IOException { - return getDirectory(name).createOutput(name); + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return getDirectory(name).createOutput(name, context); } @Override @@ -145,7 +147,7 @@ } @Override - public IndexInput openInput(String name) throws IOException { - return getDirectory(name).openInput(name); + public IndexInput openInput(String name, IOContext context) throws IOException { + return getDirectory(name).openInput(name, context); } } Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 1136453) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.Future; +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.ThreadInterruptedException; import org.apache.lucene.util.Constants; @@ -286,7 +287,7 @@ /** Creates an IndexOutput for the file with the given name. */ @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); ensureCanWrite(name); @@ -321,9 +322,9 @@ // Inherit javadoc @Override - public IndexInput openInput(String name) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return openInput(name, BufferedIndexInput.BUFFER_SIZE); + return openInput(name, context); } @Override Index: src/java/org/apache/lucene/util/BitVector.java =================================================================== --- src/java/org/apache/lucene/util/BitVector.java (revision 1136453) +++ src/java/org/apache/lucene/util/BitVector.java (working copy) @@ -19,6 +19,9 @@ import java.io.IOException; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -162,8 +165,8 @@ /** Writes this vector to the file name in Directory d, in a format that can be read by the constructor {@link #BitVector(Directory, String)}. */ - public final void write(Directory d, String name) throws IOException { - IndexOutput output = d.createOutput(name); + public final void write(Directory d, String name, IOContext context) throws IOException { + IndexOutput output = d.createOutput(name, context); try { if (isSparse()) { writeDgaps(output); // sparse bit-set more efficiently saved as d-gaps. @@ -220,8 +223,8 @@ /** Constructs a bit vector from the file name in Directory d, as written by the {@link #write} method. */ - public BitVector(Directory d, String name) throws IOException { - IndexInput input = d.openInput(name); + public BitVector(Directory d, String name, IOContext context) throws IOException { + IndexInput input = d.openInput(name, context); try { size = input.readInt(); // read size if (size == -1) { Index: src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; @@ -82,8 +83,8 @@ } @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { - final IndexInput in = dir.openInput(fileName, readBufferSize); + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + final IndexInput in = dir.openInput(fileName, context); final int baseBlockSize = in.readInt(); return new VariableIntBlockIndexInput(in) { @@ -107,7 +108,7 @@ @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { - final IndexOutput out = dir.createOutput(fileName); + final IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT_IOCONTEXT); boolean success = false; try { out.writeInt(baseBlockSize); @@ -182,7 +183,7 @@ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - state.readBufferSize, + state.context, new MockIntFactory(baseBlockSize), state.codecId); TermsIndexReaderBase indexReader; @@ -193,7 +194,7 @@ state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -208,7 +209,7 @@ state.fieldInfos, state.segmentInfo.name, postingsReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; Index: src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; @@ -84,8 +85,8 @@ } @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { - return new FixedIntBlockIndexInput(dir.openInput(fileName, readBufferSize)) { + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) { @Override protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { @@ -103,7 +104,7 @@ @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { - IndexOutput out = dir.createOutput(fileName); + IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT_IOCONTEXT); boolean success = false; try { FixedIntBlockIndexOutput ret = new FixedIntBlockIndexOutput(out, blockSize) { @@ -160,7 +161,7 @@ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - state.readBufferSize, + state.context, new MockIntFactory(blockSize), state.codecId); TermsIndexReaderBase indexReader; @@ -170,7 +171,8 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId); + BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId, + IOContext.DEFAULT_IOCONTEXT); success = true; } finally { if (!success) { @@ -185,7 +187,7 @@ state.fieldInfos, state.segmentInfo.name, postingsReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; Index: src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (working copy) @@ -93,7 +93,7 @@ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - state.readBufferSize, new MockSingleIntFactory(), state.codecId); + state.context, new MockSingleIntFactory(), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -103,7 +103,7 @@ state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -118,7 +118,7 @@ state.fieldInfos, state.segmentInfo.name, postingsReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; Index: src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.sep.IntIndexOutput; import java.io.IOException; @@ -36,7 +37,7 @@ final static int VERSION_CURRENT = VERSION_START; public MockSingleIntIndexOutput(Directory dir, String fileName) throws IOException { - out = dir.createOutput(fileName); + out = dir.createOutput(fileName, IOContext.DEFAULT_IOCONTEXT); boolean success = false; try { CodecUtil.writeHeader(out, CODEC, VERSION_CURRENT); Index: src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.store.Directory; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.sep.IntStreamFactory; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; @@ -27,8 +28,8 @@ /** @lucene.experimental */ public class MockSingleIntFactory extends IntStreamFactory { @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { - return new MockSingleIntIndexInput(dir, fileName, readBufferSize); + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return new MockSingleIntIndexInput(dir, fileName, context); } @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { Index: src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; @@ -35,9 +36,9 @@ public class MockSingleIntIndexInput extends IntIndexInput { private final IndexInput in; - public MockSingleIntIndexInput(Directory dir, String fileName, int readBufferSize) + public MockSingleIntIndexInput(Directory dir, String fileName, IOContext context) throws IOException { - in = dir.openInput(fileName, readBufferSize); + in = dir.openInput(fileName, context); CodecUtil.checkHeader(in, MockSingleIntIndexOutput.CODEC, MockSingleIntIndexOutput.VERSION_START, MockSingleIntIndexOutput.VERSION_START); Index: src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (working copy) @@ -25,6 +25,7 @@ import java.util.Set; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; @@ -103,14 +104,14 @@ } @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { // Must only use extension, because IW.addIndexes can // rename segment! final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size()); if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName); } - return f.openInput(dir, fileName, readBufferSize); + return f.openInput(dir, fileName, context); } @Override @@ -140,7 +141,7 @@ } final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT); - final IndexOutput out = state.directory.createOutput(seedFileName); + final IndexOutput out = state.directory.createOutput(seedFileName, state.context); try { out.writeLong(seed); } finally { @@ -241,7 +242,7 @@ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, SEED_EXT); - final IndexInput in = state.dir.openInput(seedFileName); + final IndexInput in = state.dir.openInput(seedFileName, state.context); final long seed = in.readLong(); if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " seed=" + seed); @@ -259,12 +260,12 @@ if (random.nextBoolean()) { postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - readBufferSize, new MockIntStreamFactory(random), state.codecId); + state.context, new MockIntStreamFactory(random), state.codecId); } else { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: reading Standard postings"); } - postingsReader = new StandardPostingsReader(state.dir, state.segmentInfo, readBufferSize, state.codecId); + postingsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); } if (random.nextBoolean()) { @@ -293,7 +294,7 @@ state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId); + state.codecId, state.context); } else { final int n2 = random.nextInt(3); if (n2 == 1) { @@ -311,7 +312,7 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - state.codecId); + state.codecId, state.context); } success = true; } finally { @@ -329,7 +330,7 @@ state.fieldInfos, state.segmentInfo.name, postingsReader, - readBufferSize, + state.context, termsCacheSize, state.codecId); success = true; Index: src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java (working copy) @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.preflex.TermInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -119,7 +120,7 @@ indexInterval = interval; fieldInfos = fis; isIndex = isi; - output = directory.createOutput(segment + (isIndex ? ".tii" : ".tis")); + output = directory.createOutput(segment + (isIndex ? ".tii" : ".tis"), IOContext.DEFAULT_IOCONTEXT); boolean success = false; try { output.writeInt(FORMAT_CURRENT); // write format Index: src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java (working copy) @@ -50,12 +50,12 @@ state.termIndexInterval); final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.FREQ_EXTENSION); - freqOut = state.directory.createOutput(freqFile); + freqOut = state.directory.createOutput(freqFile, state.context); totalNumDocs = state.numDocs; if (state.fieldInfos.hasProx()) { final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.PROX_EXTENSION); - proxOut = state.directory.createOutput(proxFile); + proxOut = state.directory.createOutput(proxFile, state.context); } else { proxOut = null; } Index: src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java =================================================================== --- src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java (revision 1136453) +++ src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java (working copy) @@ -51,7 +51,7 @@ // Whenever IW opens readers, eg for merging, we have to // keep terms order in UTF16: - return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.readBufferSize, state.termsIndexDivisor) { + return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { @Override protected boolean sortTermsByUnicode() { // We carefully peek into stack track above us: if Index: src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java =================================================================== --- src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (revision 1136453) +++ src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (working copy) @@ -31,6 +31,7 @@ import java.util.Random; import java.util.Set; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.LuceneTestCase; @@ -196,7 +197,7 @@ long length = fileLength(name); byte[] zeroes = new byte[256]; long upto = 0; - IndexOutput out = delegate.createOutput(name); + IndexOutput out = delegate.createOutput(name, IOContext.DEFAULT_IOCONTEXT); while(upto < length) { final int limit = (int) Math.min(length-upto, zeroes.length); out.writeBytes(zeroes, 0, limit); @@ -205,7 +206,7 @@ out.close(); } else if (count % 3 == 2) { // Truncate the file: - IndexOutput out = delegate.createOutput(name); + IndexOutput out = delegate.createOutput(name, IOContext.DEFAULT_IOCONTEXT); out.setLength(fileLength(name)/2); out.close(); } @@ -337,7 +338,7 @@ } @Override - public synchronized IndexOutput createOutput(String name) throws IOException { + public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException { maybeYield(); if (crashed) throw new IOException("cannot createOutput after crash"); @@ -372,7 +373,7 @@ } //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); - IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name); + IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, IOContext.DEFAULT_IOCONTEXT), name); addFileHandle(io, name, false); openFilesForWrite.add(name); @@ -401,7 +402,7 @@ } @Override - public synchronized IndexInput openInput(String name) throws IOException { + public synchronized IndexInput openInput(String name, IOContext context) throws IOException { maybeYield(); if (!delegate.fileExists(name)) throw new FileNotFoundException(name); @@ -412,7 +413,7 @@ throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); } - IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name)); + IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, IOContext.DEFAULT_IOCONTEXT)); addFileHandle(ii, name, true); return ii; } @@ -637,9 +638,9 @@ } @Override - public synchronized void copy(Directory to, String src, String dest) throws IOException { + public synchronized void copy(Directory to, String src, String dest, IOContext context) throws IOException { maybeYield(); - delegate.copy(to, src, dest); + delegate.copy(to, src, dest, IOContext.DEFAULT_IOCONTEXT); } } Index: src/test-framework/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision 1136453) +++ src/test-framework/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -1053,7 +1053,7 @@ public static MockDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); for (String file : d.listAll()) { - d.copy(impl, file, file); + d.copy(impl, file, file, IOContext.DEFAULT_IOCONTEXT); } MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); stores.put(dir, Thread.currentThread().getStackTrace());